diff --git a/.github/workflows/cmake_build.yml b/.github/workflows/cmake_build.yml index ee56c13..4f817da 100644 --- a/.github/workflows/cmake_build.yml +++ b/.github/workflows/cmake_build.yml @@ -1,10 +1,10 @@ -name: CMake build and test +name: Build and Test on: push: - branches: [ dev ] + branches: [dev] pull_request: - branches: [ dev ] + branches: [main] env: # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index adf4491..6b5ebb4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,10 +6,9 @@ name: Docs # events but only for the master branch on: push: - branches: [ dev ] + branches: [dev] pull_request: - branches: [ dev ] - + branches: [main] # A workflow run is made up of one or more jobs that can run sequentially or in parallel diff --git a/.gitmodules b/.gitmodules index 94fc651..d8da1e4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "doxygen-awesome-css"] path = doxygen-awesome-css url = https://github.com/jothepro/doxygen-awesome-css.git -[submodule "lib/googletest"] - path = lib/googletest - url = https://github.com/google/googletest.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 8684f69..809883d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,16 +1,17 @@ cmake_minimum_required(VERSION 3.0) project(Signapse) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread") add_subdirectory(src) add_subdirectory(test) -enable_testing() -add_test(camera_unit_test camera_test) -#add_test(inference_test inference_test) - set_target_properties(Signapse PROPERTIES RUNTIME_OUTPUT_DIRECTORY ../bin/) set_property(TARGET Signapse PROPERTY AUTOMOC ON) -#set_target_properties(camera_test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) -#set_target_properties(inference_test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) -add_compile_options(-Wall) \ No newline at end of file +add_compile_options(-Wall) + +#allows google-tests to be called with ctest (not recommended as output will be less verbose, but in-case the user insists on using ctest) +enable_testing() + +add_test( + NAME test + COMMAND ./test/main +) diff --git a/README.md b/README.md index fd94a3f..129d7c7 100644 --- a/README.md +++ b/README.md @@ -1,74 +1,94 @@
+

Signapse

+ logo +
+ Signapse is here to help you learn and practice sign language on an embedded device you can take anywhere! +
+ πŸ“° 🀩 As featured on: + Hackster.io and + Channel 969!! 🀩 πŸ“° +
+
![Contributors](https://img.shields.io/github/contributors/albanjoseph/Signapse?style=for-the-badge) ![GitHub Repo stars](https://img.shields.io/github/stars/albanjoseph/Signapse?style=for-the-badge) ![Issues](https://img.shields.io/github/issues-raw/albanjoseph/Signapse?style=for-the-badge) [![License](https://img.shields.io/github/license/albanjoseph/Signapse?style=for-the-badge)](https://github.com/albanjoseph/Signapse/blob/main/LICENSE) - +
+ Report Bug + . + Request Feature +
+ ![Build & Test](https://github.com/albanjoseph/Signapse/actions/workflows/cmake_build.yml/badge.svg) ![Docs](https://github.com/albanjoseph/Signapse/actions/workflows/docs.yml/badge.svg) - Signapse Logo - - -

Signapse

-

+ πŸŽ₯ Check out our product + Demo Video!!
- πŸ“° 🀩 As featured on: - Hackster.io and - Channel 969!! 🀩 πŸ“° + πŸ“œ Explore our + Wiki to learn more!!
-
- Report Bug - Β· - Request Feature + πŸ–‡οΈ Browse the + Developer Documentation!!

+
+

-

+# In a Nutshell 🌰 -# About Signapse +**Signapse is free open source software that helps everyday people learn sign language for free!​** -**Signapse is an open source software tool that helps everyday people learn sign language for free!​** +Our deep learning enhanced video pipeline is able to detect when users are making the correct input sign, this is used to challenge users to make example signs shown on-screen. At present Signapse is configured to teach the American Sign Language character alphabet; although we hope to add support for more complex and varied hand signs in the future. -For more information check out our Wiki where you can find our installation guide, developer documentation and more. Or to get started quickly simply watch our install guide and demo video below. +For installation information check out our installation guide; or to get started quickly simply watch our install guide and demo video below. For best results using Signapse, we reccomend keeping your hand witin the green box shown on screen and signing in a well lit environment with a plain background; this helps our AI system easily detect your hand signs. Happy Signapsing! πŸ₯³ https://user-images.githubusercontent.com/33161910/163428318-66b07cc5-5431-41e9-8261-a6f18a6c4551.mp4 -# Social Media -Keep up to speed with all of the Signapse news! +# Social Media 🌐 +Keep up to date with Signapse! Find us on all our social channels: +
+

signupsΒ· signapse Β· youtubeΒ· -tiktok +tiktok Β· + instagram +

-# Technologies -

cplusplus - rpi - linux +# Technologies βš™οΈ +Signapse is built using: +- [C++ Programming Language](https://www.cplusplus.com/) +- [Debian/Ubuntu Linux](https://www.linux.org/) +- [Raspberry Pi](https://www.raspberrypi.org) +- [Tensorflow](https://www.tensorflow.org/) +- [OpenCV](https://opencv.org/) +- [Google Test](https://github.com/google/googletest) +- [Doxygen](https://www.doxygen.nl/index.html) +- [Qt](https://www.qt.io/) -# License +# License πŸ“° Distributed under the GPL-3.0 License. See [`LICENSE`](https://github.com/albanjoseph/Signapse/blob/main/LICENSE) for more information. -# Contact Us +# Contact Us πŸ“ž - πŸ”­This project is being completed by a team of students at the University of Glasgow : - * [Adam Frew](https://github.com/Saweenbarra) * [Alban Joseph](https://github.com/albanjoseph) * [Lewis Russell](https://github.com/charger4241) * [Ross Gardiner](https://github.com/rossGardiner) -- πŸ“« Contact us: **signapse.glasgow@gmail.com** +- πŸ“« Email: **signapse.glasgow@gmail.com** diff --git a/dconfig b/dconfig index 2ec017d..9275802 100644 --- a/dconfig +++ b/dconfig @@ -963,7 +963,7 @@ RECURSIVE = NO # Note that relative paths are relative to the directory from which doxygen is # run. -EXCLUDE = +EXCLUDE = src/MultiThreadedSchedulableLink.cpp src/MultiThreadedSchedulableLink.h src/BatchCNNProcessor.cpp src/BatchCNNProcessor.h src/NThreadedCNNProcessor.cpp src/NThreadedCNNProcessor.h src/QtGeneratedGui.h # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded diff --git a/images/classSceneCallback__inherit__graph.png b/images/classSceneCallback__inherit__graph.png new file mode 100644 index 0000000..6043200 Binary files /dev/null and b/images/classSceneCallback__inherit__graph.png differ diff --git a/images/pipeline.pdf b/images/pipeline.pdf new file mode 100644 index 0000000..82d0f62 Binary files /dev/null and b/images/pipeline.pdf differ diff --git a/images/pipeline.png b/images/pipeline.png new file mode 100644 index 0000000..e302508 Binary files /dev/null and b/images/pipeline.png differ diff --git a/images/pipeline2.png b/images/pipeline2.png new file mode 100644 index 0000000..99e89ac Binary files /dev/null and b/images/pipeline2.png differ diff --git a/setup.sh b/setup.sh index 3daa1e7..9820f09 100644 --- a/setup.sh +++ b/setup.sh @@ -1,8 +1,6 @@ -sudo apt-get update && sudo apt-get upgrade -y sudo apt install cmake gcc qtbase5-dev qtdeclarative5-dev qt5-default libgtest-dev -y CV_VERSION="69357b1e88680658a07cffde7678a4d697469f03" - ## Download OpenCV code printf "Downloading OpenCV ..." if [ ! -d "opencv_src" ]; then @@ -13,17 +11,17 @@ if [ ! -d "opencv_src" ]; then else printf " skipped\n" fi - -## Build OpenCV -printf "Building OpenCV ..." if [ ! -d "opencv_build" ]; then printf "\n" mkdir -p opencv_build cd opencv_build - cmake -D WITH_QT=ON -DCMAKE_BUILD_TYPE=Release ../opencv_src || exit 1 - make || exit 1 + #install with only the nessessary packages - reduces build time + cmake -DBUILD_LIST=dnn,improc,videoio, -DCMAKE_BUILD_TYPE=Release ../opencv_src || exit 1 + #set make jobs to 4 - reduces build time + make -j4 || exit 1 cd .. else printf " skipped\n" fi + diff --git a/src/BlockingQueue.cpp b/src/BlockingQueue.cpp index daf0391..a7c3f64 100644 --- a/src/BlockingQueue.cpp +++ b/src/BlockingQueue.cpp @@ -2,7 +2,11 @@ // Blocking queue implementation, currently only blocks when Get is attempted while there are no elements inside the queue. In the future and max size could be defined and blocking could be added when the queue is full // #include "BlockingQueue.h" - +/*! + * Pop method, uses mutex lock to sleep the current thread until data are available on the queue. This may be used to synchonise and schedule threads. + * @tparam T + * @return Element on deque back + */ template T BlockingQueue::Pop() { std::unique_lock lock(mutex); @@ -11,16 +15,34 @@ T BlockingQueue::Pop() { internalQueue.pop_back(); return ret; } - +/*! + * + * @tparam T + * @return true if internal queue empty, false otherwise + */ template bool BlockingQueue::IsEmpty(){ return internalQueue.empty(); } +/*! + * Gets the current length of elements in the queue. + * @tparam T + * @return size of internal deque + */ +template +int BlockingQueue::Size(){ + return internalQueue.size(); +} + +/*! + * adds element to the queue. + * @tparam T + * @param toPush + */ template void BlockingQueue::Push(T toPush) { { - std::unique_lock lock(mutex); internalQueue.push_front(toPush); } condition.notify_all(); diff --git a/src/BlockingQueue.h b/src/BlockingQueue.h index 3866332..d7005a6 100644 --- a/src/BlockingQueue.h +++ b/src/BlockingQueue.h @@ -7,7 +7,7 @@ #include #include #include -#include "scene.h" +#include "Scene.h" template @@ -15,16 +15,17 @@ template /*! */ + +/*! + * Class to wrap around std::deque and block thread execution when no data is available at the output. + * @tparam T Type of elements in the queue + */ class BlockingQueue { public: - //! Public member function - /*! - - */ void Push(T toPush); T Pop(); bool IsEmpty(); - + int Size(); private: std::deque internalQueue; std::mutex mutex; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dfd12a2..36a1991 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,25 +1,31 @@ +#find and include opencv set(OpenCV_DIR "../opencv_build") -message("Linking OpenCV build at directory: " ${OpenCV_DIR}$ ) find_package(OpenCV REQUIRED) include_directories( ${OpenCV_INCLUDE_DIRS} ) -message("Including OpenCV build at directory: " ${OpenCV_INCLUDE_DIRS}) +#find Qt5Widgets find_package(Qt5Widgets REQUIRED) -add_executable(Signapse main.cpp reel.cpp camera.cpp taskmaster.cpp CNNProcessor.cpp BlockingQueue.cpp Gui.cpp ProgressBar.cpp) -target_link_libraries(Signapse ${OpenCV_LIBS} Qt5::Widgets) +#find Threads, prefer pthread +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) +set(THREADS_PREFER_PTHREAD_FLAG TRUE) +find_package(Threads REQUIRED) -add_library(reel reel.cpp reel.h) -target_link_libraries(reel ${OpenCV_LIBS}) +#compile and link main signapse executable +add_executable(Signapse main.cpp Camera.cpp CNNProcessor.cpp PipelineLink.cpp LinkSplitter.cpp SchedulableLink.cpp PreProcessor.cpp BlockingQueue.cpp Gui.cpp ProgressBar.cpp ) +target_link_libraries(Signapse ${OpenCV_LIBS} Qt5::Widgets Threads::Threads) -add_library(camera camera.cpp camera.h) -target_link_libraries(camera ${OpenCV_LIBS} reel) +#build objects for testing as static libs +add_library(Camera Camera.cpp Camera.h) +target_link_libraries(Camera ${OpenCV_LIBS} Threads::Threads) -add_library(CNNProcessor CNNProcessor.cpp CNNProcessor.h) -target_link_libraries(CNNProcessor ${OpenCV_LIBS}) +add_library(CNNProcessor CNNProcessor.cpp CNNProcessor.h SchedulableLink.cpp PipelineLink.cpp) +target_link_libraries(CNNProcessor ${OpenCV_LIBS} Threads::Threads) -add_library(FrameEditor FrameEditor.cpp FrameEditor.h) -target_link_libraries(FrameEditor ${OpenCV_LIBS}) +add_library(PreProcessor PreProcessor.cpp PreProcessor.h) +target_link_libraries(PreProcessor ${OpenCV_LIBS}) + +add_library(BlockingQueue BlockingQueue.cpp BlockingQueue.h) add_library(Gui Gui.cpp Gui.h) -target_link_libraries(Gui ${OpenCV_LIBS} Qt5::Widgets) \ No newline at end of file +target_link_libraries(Gui ${OpenCV_LIBS} Qt5::Widgets Threads::Threads) diff --git a/src/CNNProcessor.cpp b/src/CNNProcessor.cpp index bf332a4..c13ca81 100644 --- a/src/CNNProcessor.cpp +++ b/src/CNNProcessor.cpp @@ -1,57 +1,46 @@ #include "CNNProcessor.h" -#include -#include -#include "scene.h" -#include "SignapseUtils.h" - +/*! + * Loads neural network from location on disk + * @param modelPath + */ void CNNProcessor::LoadModel(std::string modelPath){ net = cv::dnn::readNetFromTensorflow(modelPath); } -void CNNProcessor::threadLoop() { - return; -} - - - -void CNNProcessor::start_thread(){ - cnnProcessorThread = std::thread(&CNNProcessor::threadLoop, this); -} - -CNNProcessor::CNNProcessor(Reel* setReadFrom, std::string modelPath){ - readFrom = setReadFrom; - LoadModel(modelPath); +/*! + * Constructor, inits settings and loads model + * @param s + */ +CNNProcessor::CNNProcessor(CNNProcessorSettings s) : settings(s) { + LoadModel(settings.ModelPath); } -CNNProcessor::CNNProcessor(std::string modelPath) { - LoadModel(modelPath); -} - -void CNNProcessor::Loop(){ - while(true){ - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - SelfPush(); - } -} +/*! + * Makes OpenCV image "blob" from the region of interest in the frame. Blob format is required for inference. + * @param scene + * @return matrix image blob, ready for inference + */ cv::Mat CNNProcessor::MakeBlob(Scene scene){ - //make blob - int x = scene.regionOfInterest[0]; int y = scene.regionOfInterest[1]; - int width = scene.regionOfInterest[2] - scene.regionOfInterest[0]; - int height = scene.regionOfInterest[3] - scene.regionOfInterest[1]; - + int x = scene.regionOfInterest.UpperLeft.x; int y = scene.regionOfInterest.UpperLeft.y; + int width = scene.regionOfInterest.LowerRight.x - x; + int height = scene.regionOfInterest.LowerRight.y - y; cv::Mat roi = scene.frame(cv::Range(y, y+height), cv::Range(x, x+width)); - cv::Mat rgb; - cv::cvtColor(roi, rgb, cv::COLOR_BGR2RGB); - cv::Mat smallRGB; - cv::resize(rgb, smallRGB, cv::Size(224,224)); + cv::Mat small; + cv::resize(roi, small, cv::Size(settings.InputDim_x,settings.InputDim_y)); cv::Mat blob; - cv::dnn::blobFromImage(smallRGB, blob, (1.0 / 255.0)); + cv::dnn::blobFromImage(small, blob, (1.0 / 255.0)); return blob; } +/*! + * Executed network inference on given scene, populates the scene with results. + * @param scene input scene + * @return output processed scene + */ Scene CNNProcessor::ProcessScene(Scene scene){ + if(scene.frame.empty()) return scene; cv::Mat blob = MakeBlob(scene); net.setInput(blob); cv::Mat prob = net.forward(); @@ -62,17 +51,5 @@ Scene CNNProcessor::ProcessScene(Scene scene){ scene.result = SignapseUtils::getLetterFromDigit(classId); return scene; } -void CNNProcessor::nextScene(Scene next) { - Scene updatedFrame = ProcessScene(next); - if(!sceneCallback) return; - sceneCallback->nextScene(updatedFrame); -} -void CNNProcessor::registerCallback(SceneCallback* scb){ - sceneCallback = scb; -} -void CNNProcessor::SelfPush() { - Scene frame = readFrom->Pop(); - sceneQueue.Push(frame); -} diff --git a/src/CNNProcessor.h b/src/CNNProcessor.h index bb299e8..21e0621 100644 --- a/src/CNNProcessor.h +++ b/src/CNNProcessor.h @@ -1,62 +1,32 @@ -// -// Created by ross on 09/02/2022. -// - #ifndef SIGNAPSE_CNNPROCESSOR_H #define SIGNAPSE_CNNPROCESSOR_H -#include "reel.h" -#include "camera.h" #include #include -#include #include -#include "SceneCallback.h" + #include +#include +#include + +#include "SchedulableLink.h" +#include "CNNProcessorSettings.h" +#include "SignapseUtils.h" -//! CNNProcessor class. /*! - Class for interfacing with convolutional neural network -*/ -class CNNProcessor : public Reel, public SceneCallback { + * Schedulable Pipeline Link which performs network inference on each Scene, populating the results field. + */ +class CNNProcessor : public SchedulableLink{ public: - virtual void nextScene(Scene next); - //! Public member function. - /*! - Loop function. - */ - void Loop(); - //! Constructor takes Reel object as argument and assigns readFrom Reel to this value. - /*! - \param setReadFrom Reel to be copied to readFrom. - */ - CNNProcessor(Reel *setReadFrom, std::string modelPath); - - CNNProcessor(std::string modelPath); - //! Public member function. - /*! - Pops Scene from readFrom Reel and pushes to CNNProcessor own sceneQueue. - */ - void SelfPush(); - - Scene ProcessScene(Scene scene); - + Scene ProcessScene(Scene s); + CNNProcessor(CNNProcessorSettings s); cv::Mat MakeBlob(Scene scene); - void start_thread(); - - void registerCallback(SceneCallback *scb); - -private: - void threadLoop(); - +protected: + CNNProcessorSettings settings; void LoadModel(std::string modelPath); - - Reel *readFrom; cv::dnn::Net net; - std::thread cnnProcessorThread; - SceneCallback *sceneCallback = nullptr; }; -#endif //SIGNAPSE_CNNPROCESSOR_H +#endif \ No newline at end of file diff --git a/src/CNNProcessorSettings.h b/src/CNNProcessorSettings.h new file mode 100644 index 0000000..6668ff6 --- /dev/null +++ b/src/CNNProcessorSettings.h @@ -0,0 +1,36 @@ +#include +#include "SignapseUtils.h" + +#define MOBNET_V2_INPUT_DIM_X 224 +#define MOBNET_V2_INPUT_DIM_Y 224 +#define MOBNET_V2_PATH "models/asl-mobilenetv2.pb" + +/*! + * A simple struct to store default configurations for CNNProcessor settings. + */ +struct CNNProcessorSettings { + CNNProcessorSettings(std::string network = "mobnetv2"){ + if(network == "mobnetv2") { + ModelPath = MOBNET_V2_PATH; + InputDim_x = MOBNET_V2_INPUT_DIM_X; + InputDim_y = MOBNET_V2_INPUT_DIM_Y; + } + else{ + ; //space for more default network setups + } + } + /*! + * Copy constructor + * @param cpy + */ + CNNProcessorSettings(const CNNProcessorSettings& cpy){ + ModelPath = cpy.ModelPath; + InputDim_x = cpy.InputDim_x; + InputDim_y = cpy.InputDim_y; + } + std::string ModelPath = ""; + int InputDim_x = 0; + int InputDim_y = 0; +}; + + diff --git a/src/Camera.cpp b/src/Camera.cpp new file mode 100644 index 0000000..d20b216 --- /dev/null +++ b/src/Camera.cpp @@ -0,0 +1,73 @@ +#include "Camera.h" + +/*! + * Camera default constructor, inits settings, sets isOn to true. + */ +Camera::Camera() : cameraSettings() { + isOn=true; +} +/*! + * Camera constructor from settings, sets isOn to true. + */ +Camera::Camera(CameraSettings settings) : cameraSettings(settings){ + isOn = true; +} + +/*! + * Sets the isOn state of the camera + * @param state + */ +void Camera::setOn(bool state){ + isOn = state; +} + +/*! + * Loops while camera is on to add frames to the pipeline + */ +void Camera::threadLoop(){ + while(isOn){ + postFrame(); + } +} + +/*! + * Gets the next available frame and passes it on to the registered callback. Relies on the videoCapture.read() OpenCV method which is understood to wait for an intra-frame delay. + */ +void Camera::postFrame(){ + if(!sceneCallback) return; + cv::Mat cap; + videoCapture.read(cap); + // check if we succeeded + if (cap.empty()) { + std::cerr << "ERROR! blank frame grabbed\n"; + return; + } + Scene s; + s.frame=cap; + sceneCallback->NextScene(s); +} + +/*! + * Starts the worker thread recording + */ +void Camera::Start(){ + videoCapture.open(cameraSettings.deviceID, cameraSettings.apiID); + cameraThread = std::thread(&Camera::threadLoop, this); +} + +/*! + * Frees thread resources and stops recording, must be called prior to program exit. + */ +void Camera::Stop(){ + isOn=false; + cameraThread.join(); + +} + +/*! + * Returns the isON state of the camera + * @return true if on; false otherwise + */ +bool Camera::getOn() { + return isOn; +} diff --git a/src/Camera.h b/src/Camera.h new file mode 100644 index 0000000..99cced2 --- /dev/null +++ b/src/Camera.h @@ -0,0 +1,37 @@ +#include +#include + +#include +#include +#include + +#include "PipelineLink.h" +#include "CameraSettings.h" + + +/*! + * Simple Pipeline element which integrates frame acquisition from the camera feed. + */ +class Camera: public PipelineLink{ +public: + Camera(); + Camera(CameraSettings settings); + bool getOn(); + void setOn(bool state); + void Start(); + void Stop(); + +private: + void postFrame(); + void threadLoop(); + cv::VideoCapture videoCapture; + CameraSettings cameraSettings; + //! Private member thread + std::thread cameraThread; + //! Private member variable containing camera object status. + /*! + false = Camera is off. + true = Camera is on. + */ + bool isOn = false; +}; diff --git a/src/CameraSettings.h b/src/CameraSettings.h new file mode 100644 index 0000000..bd45de4 --- /dev/null +++ b/src/CameraSettings.h @@ -0,0 +1,25 @@ +#ifndef SIGNAPSE_CAMERASETTINGS_H +#define SIGNAPSE_CAMERASETTINGS_H + + + +struct CameraSettings{ + CameraSettings(int dID=0){ + deviceID = dID; + } + CameraSettings(const CameraSettings& cpy){ + deviceID = cpy.deviceID; + apiID = cpy.apiID; + } + /*! + * Member containing device ID, may be modified via the constructor; assigned to 0 by default + */ + int deviceID = 0; + //! Member variable containing web cam API ID. + /*! + 0 = autodetect default API + */ + int apiID = 0; +}; + +#endif //SIGNAPSE_CAMERASETTINGS_H diff --git a/src/FrameEditor.cpp b/src/FrameEditor.cpp deleted file mode 100644 index 30246cf..0000000 --- a/src/FrameEditor.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// -// Created by ross on 24/03/2022. -// -#include -#include -#include "FrameEditor.h" - - - -cv::Mat FrameEditor::drawBox(cv::Mat img, int *box) { - int x = box[0]; int y = box[1]; - int width = box[2] - box[0]; int height = box[3] - box[1]; - cv::Rect rect(x, y, width, height); - cv::rectangle(img, rect, cv::Scalar(0, 255, 0)); - return img; -} -Scene FrameEditor::drawBox(Scene s) { - cv::Mat temp = s.frame; - int x = s.regionOfInterest[0]; int y = s.regionOfInterest[1]; - int width = s.regionOfInterest[2] - s.regionOfInterest[0]; - int height = s.regionOfInterest[3] - s.regionOfInterest[1]; - cv::Rect rect(x, y, width, height); - cv::rectangle(temp, rect, cv::Scalar(0,255,0)); - Scene ret = s; - s.frame = temp; - return s; -} diff --git a/src/FrameEditor.h b/src/FrameEditor.h deleted file mode 100644 index 32629c8..0000000 --- a/src/FrameEditor.h +++ /dev/null @@ -1,17 +0,0 @@ -// -// Created by ross on 24/03/2022. -// - -#ifndef SIGNAPSE_FRAMEEDITOR_H -#define SIGNAPSE_FRAMEEDITOR_H -#include -#include "scene.h" - - -class FrameEditor { -public: - static cv::Mat drawBox(cv::Mat img, int box[4]); - static Scene drawBox(Scene s); -}; - -#endif //SIGNAPSE_FRAMEEDITOR_H diff --git a/src/Gui.cpp b/src/Gui.cpp index 17d8dab..730eee8 100644 --- a/src/Gui.cpp +++ b/src/Gui.cpp @@ -1,64 +1,93 @@ -// -// Created by ross on 01/04/2022. -// - #include "Gui.h" #define UI_WDH 1000 #define UI_HGT 700 -//virtual Gui::~Gui() {}; - -Gui::Gui() { - widget = new QMainWindow(); +/*! + * Constructor sets up GUI, makes signal connections + * @param win points to QMainWindow from main thread + * @param ui_win points to Ui_MainWindow from main thread + */ +Gui::Gui(QMainWindow* win, Ui_MainWindow* ui_win) { + widget = win; widget->setFixedSize(UI_WDH, UI_HGT); - ui = new Ui_MainWindow(); + ui = ui_win; ui->setupUi(widget); - SetTargetImage("A"); + SetTask("A"); makeConnections(); + SignapseUtils::randSeed(); } - -void Gui::nextScene(Scene next) { - cv::Mat img = next.frame; - QImage imgIn= QImage((uchar*) img.data, img.cols, img.rows, img.step, QImage::Format_RGB888); - ui->label->setPixmap(QPixmap::fromImage(imgIn)); - ui->label->resize(ui->label->pixmap()->size()); - int progress = progress_bar.get_progress(next, current_task); - emit progressChanged(progress); +/*! + * Handles the next scene from the video processing pipeline. If the result is empty, the viewing pane is updated, otherwise, the progress bar is updated. + * @param next The Scene to be processed + */ +void Gui::NextScene(Scene next) { + //flip frame + if(next.result == "") { + cv::Mat temp; + cv::flip(next.frame, temp, 1); + QImage imgIn = QImage((uchar *) temp.data, temp.cols, temp.rows, temp.step, + QImage::Format_RGB888); + ui->label->setPixmap(QPixmap::fromImage(imgIn)); + ui->label->resize(ui->label->pixmap()->size()); + } + else{ + int progress = progressBar.GetProgress(next.result, currentTask); + emit progressChanged(progress); + if(progress >= 100){ + buttonPressed(); + } + } + } - +/*! + * Sets underlying widget visibility. + * @param visible + */ void Gui::SetVisible(bool visible) { widget->setVisible(visible); } - - -void Gui::SetTargetImage(int target) { - std::string letter = SignapseUtils::getLetterFromDigit(target); - SetTargetImage(letter); -} - +/*! + * Sets the target to match with user sign + * @param letter string represenation of the target sign + */ void Gui::SetTargetImage(std::string letter) { std::string impath = testFolder + letter + "_test.jpg"; cv::Mat img = cv::imread(impath); setDemoImage(img); setTaskText(letter); } -void Gui::ButtonPressed(){ - std::string new_task = SignapseUtils::getLetterFromDigit(SignapseUtils::makeTask()); +/*! + * Method handler for when the next task button is pressed, sets new task and resets the progress bar. + */ +void Gui::buttonPressed(){ + std::string new_task = SignapseUtils::makeTask(); SetTargetImage(new_task); - current_task = *new_task.c_str(); - progress_bar.reset_progress(); + currentTask = new_task; + progressBar.ResetProgress(); +} + +void Gui::updateThreshold(){ + progressBar.SetThreshold(ui->spinBox->value()); + progressBar.ResetProgress(); } +/*! + * Makes signal connections for the GUI interrupts. + */ void Gui::makeConnections() { - QObject::connect(ui->pushButton, &QPushButton::released, this, &Gui::ButtonPressed); + QObject::connect(ui->pushButton, &QPushButton::released, this, &Gui::buttonPressed); QObject::connect(this, &Gui::progressChanged, ui->progressBar, &QProgressBar::setValue); + QObject::connect(ui->spinBox, QOverload::of(&QSpinBox::valueChanged), this, &Gui::updateThreshold); } + void Gui::setDemoImage(cv::Mat img) { cv::Mat rgb; cv::cvtColor(img, rgb, cv::COLOR_BGR2RGB); - ui->label_2->setPixmap(QPixmap::fromImage(QImage(rgb.data, rgb.cols, rgb.rows, rgb.step, QImage::Format_RGB888))); + cv::Mat flip; + cv::flip(rgb, flip, 1); + ui->label_2->setPixmap(QPixmap::fromImage(QImage(flip.data, flip.cols, flip.rows, flip.step, QImage::Format_RGB888))); ui->label_2->resize(ui->label_2->pixmap()->size()); } @@ -67,6 +96,7 @@ void Gui::setTaskText(std::string letter){ ui->listWidget->item(2)->setText(QCoreApplication::translate("MainWindow", letter.c_str(), nullptr)); } -void Gui::set_task(char new_task){ - current_task = new_task; +void Gui::SetTask(std::string newTask){ + currentTask = newTask; + SetTargetImage("A"); } \ No newline at end of file diff --git a/src/Gui.h b/src/Gui.h index a28b6fd..d187f03 100644 --- a/src/Gui.h +++ b/src/Gui.h @@ -1,52 +1,48 @@ -// -// Created by ross on 01/04/2022. -// - #ifndef SIGNAPSE_GUI_H #define SIGNAPSE_GUI_H + #include -#include "qtGenGui.h" +#include "QtGeneratedGui.h" #include "stdlib.h" #include -#include #include -#include "ProgressBar.h" - - +#include +#include "ProgressBar.h" #include "SignapseUtils.h" #include "SceneCallback.h" #define testFolder "test/asl_alphabet_test/" /* - * A class to wrap the QT generated header file and handle GUI functionality + * A class to wrap the QT generated header file and handle GUI functionality. + * Inherits from QWidget and SceneCallback */ class Gui : public QWidget, public SceneCallback{ Q_OBJECT public: - //Gui (QObject *_parent); - virtual void nextScene(Scene next); - Gui(); + virtual void NextScene(Scene next); + Gui(QMainWindow* win, Ui_MainWindow* ui_win); void SetVisible(bool visible); void SetTargetImage(int target); void SetTargetImage(std::string target); - void ButtonPressed(); - void set_task(char new_task); + void SetTask(std::string newTask); + void buttonPressed(); + void updateThreshold(); private: QMainWindow *widget; Ui_MainWindow *ui; - ProgressBar progress_bar; + ProgressBar progressBar; void setDemoImage(cv::Mat img); void setTaskText(std::string letter); void makeConnections(); - char current_task; + std::string currentTask; signals: void progressChanged(int progress); + }; - -#endif //SIGNAPSE_GUI_H +#endif diff --git a/src/LinkSplitter.cpp b/src/LinkSplitter.cpp new file mode 100644 index 0000000..c50ac3e --- /dev/null +++ b/src/LinkSplitter.cpp @@ -0,0 +1,17 @@ +#include "LinkSplitter.h" + +/*! + * Override of NextScene to pass scene reference to a maximum of two registered callbacks + * @param s scene + */ +void LinkSplitter::NextScene(Scene s) { + if(sceneCallback) sceneCallback->NextScene(s); + if(secondarySceneCallback) secondarySceneCallback->NextScene(s); +} +/*! + * Used to register a secondary callback. + * @param scb callback to register. + */ +void LinkSplitter::RegisterSecondaryCallback(SceneCallback *scb) { + secondarySceneCallback = scb; +} \ No newline at end of file diff --git a/src/LinkSplitter.h b/src/LinkSplitter.h new file mode 100644 index 0000000..070b6d2 --- /dev/null +++ b/src/LinkSplitter.h @@ -0,0 +1,20 @@ +#ifndef SIGNAPSE_DOUBLESCENELINKER_H +#define SIGNAPSE_DOUBLESCENELINKER_H + +#include "PipelineLink.h" + +/*! + * A pipeline element which extends PipelineLink to add a secondary callback. The NextScene function is overridden to pass on the scene reference to both registered callbacks. + * N.B. this does not duplicate the scene, merely copies the reference to another element. No extra memory is allocated! + */ +class LinkSplitter : public PipelineLink{ +public: + void NextScene(Scene s); + void RegisterSecondaryCallback(SceneCallback* scb); +public: + SceneCallback* secondarySceneCallback = nullptr; + +}; + + +#endif //SIGNAPSE_DOUBLESCENELINKER_H diff --git a/src/PipelineLink.cpp b/src/PipelineLink.cpp new file mode 100644 index 0000000..7cecbb7 --- /dev/null +++ b/src/PipelineLink.cpp @@ -0,0 +1,17 @@ +#include "PipelineLink.h" + +/*! + * Used to register the next pipeline element. + * @param scb reference to the next pipeline element + */ +void PipelineLink::RegisterCallback(SceneCallback *scb) { + sceneCallback = scb; +} +/*! + * Default NextScene implementation, simply passes the scene on to the registered callback + * @param scene + */ +void PipelineLink::NextScene(Scene scene) { + if(!sceneCallback) return; + sceneCallback->NextScene(scene); +} \ No newline at end of file diff --git a/src/PipelineLink.h b/src/PipelineLink.h new file mode 100644 index 0000000..f0ab32d --- /dev/null +++ b/src/PipelineLink.h @@ -0,0 +1,19 @@ +#ifndef SIGNAPSE_SCENELINKER_H +#define SIGNAPSE_SCENELINKER_H + +#include "SceneCallback.h" + +/*! + * A simple class which extends SceneCallback to add a RegisterCallback method. Also implements a default NextScene function. + * This class forms the basis for pipeline elements in the Signapse video processing architecture. + */ +class PipelineLink : public SceneCallback{ +public: + void RegisterCallback(SceneCallback* scb); + +protected: + void NextScene(Scene scene); + SceneCallback* sceneCallback = nullptr; +}; + +#endif \ No newline at end of file diff --git a/src/PreProcessor.cpp b/src/PreProcessor.cpp new file mode 100644 index 0000000..9b7e27c --- /dev/null +++ b/src/PreProcessor.cpp @@ -0,0 +1,82 @@ +#include +#include + +#include "PreProcessor.h" + +/*! + * Handles processing the given scene and calling back to next pipeline element + * @param scene + */ +void PreProcessor::NextScene(Scene scene){ + cv::Size sz = scene.frame.size(); + scene.regionOfInterest = BoundingBox((int)(sz.width * settings.relativeBoundingBox[0]), + (int)(sz.height * settings.relativeBoundingBox[1]), + (int)(sz.width * settings.relativeBoundingBox[2]), + (int)(sz.height * settings.relativeBoundingBox[3])); + Scene out = switchRGB2BGR(scene); + out = drawBox(scene); + if(!sceneCallback) return; + sceneCallback->NextScene(out); +} +/*! + * + * @param PreProcessorSettings for initialisation + */ +PreProcessor::PreProcessor(PreProcessorSettings s) : settings(s) {} + +/*! + * Default constructor; inits default settings + */ +PreProcessor::PreProcessor() : settings() {} + +/*! + * Processing method to switch the pixel format of the internal frame. Performed with standard OpenCV functions + * @param s scene + * @return scene with pixel format rearranged + */ +Scene PreProcessor::switchRGB2BGR(Scene s) { + cv::Mat temp = s.frame; + cv::cvtColor(temp, s.frame, cv::COLOR_BGR2RGB); + return s; +} + + +/*! + * Adds a green rectangle into the internal scene frame, highlighting the region of interest. This shows the user where to put their hand sign to interface with the neural network. + * @param s scene for processing + * @return the processed scene + */ +Scene PreProcessor::drawBox(Scene s) { + cv::Mat temp = s.frame; + int x = s.regionOfInterest.UpperLeft.x; int y = s.regionOfInterest.UpperLeft.y; + int width = s.regionOfInterest.LowerRight.x - x; + int height = s.regionOfInterest.LowerRight.y - y; + cv::Rect rect(x, y, width, height); + cv::rectangle(temp, rect, cv::Scalar(0,255,0)); + Scene ret = s; + s.frame = temp; + return s; +} + +/*! + * Used to set the bounding box each scene is marked with. Set as float values in range [0-1] indicating fractional coordinates in the image frame. + * @param upperLeftX + * @param upperLeftY + * @param lowerRightX + * @param lowerRightY + */ +void PreProcessor::SetBoundingBox(float upperLeftX, float upperLeftY, float lowerRightX, float lowerRightY) { + settings.relativeBoundingBox[0] = upperLeftX; + settings.relativeBoundingBox[1] = upperLeftY; + settings.relativeBoundingBox[2] = lowerRightX; + settings.relativeBoundingBox[3] = lowerRightY; + for(int i = 0; i < 4; i++){ + float val = settings.relativeBoundingBox[i]; + if(val > 1.0f){ + settings.relativeBoundingBox[i] = 1.0f; + } + else if (val < 0.0f){ + settings.relativeBoundingBox[i] = 0.0f; + } + } +} diff --git a/src/PreProcessor.h b/src/PreProcessor.h new file mode 100644 index 0000000..a44f970 --- /dev/null +++ b/src/PreProcessor.h @@ -0,0 +1,20 @@ +#include +#include "Scene.h" +#include "PipelineLink.h" +#include "PreProcessorSettings.h" +/*! + * A pipeline element to perform pre-processing required for Signapse. Adds region of interest, bounding box rectangle and switches pixel format for neural network. + */ +class PreProcessor : public PipelineLink{ +public: + PreProcessor(); + PreProcessor(PreProcessorSettings settings); + void NextScene(Scene scene); + void SetBoundingBox(float upperLeftX, float upperLeftY, float lowerRightX, float lowerRightY); + +private: + Scene drawBox(Scene s); + Scene switchRGB2BGR(Scene s); + PreProcessorSettings settings; +}; + diff --git a/src/PreProcessorSettings.h b/src/PreProcessorSettings.h new file mode 100644 index 0000000..232cd2e --- /dev/null +++ b/src/PreProcessorSettings.h @@ -0,0 +1,30 @@ + +#define X0 0.25f +#define Y0 0.25f +#define X1 0.75f +#define Y1 0.75f +/*! + * A simple struct to add some default settings for PreProcessor + */ +struct PreProcessorSettings{ + /*! + * Default constructor + */ + PreProcessorSettings(){ + relativeBoundingBox[0] = X0; + relativeBoundingBox[1] = Y0; + relativeBoundingBox[2] = X1; + relativeBoundingBox[3] = Y1; + } + /*! + * Copy constructor + * @param cpy + */ + PreProcessorSettings(const PreProcessorSettings& cpy){ + relativeBoundingBox[0] = cpy.relativeBoundingBox[0]; + relativeBoundingBox[1] = cpy.relativeBoundingBox[1]; + relativeBoundingBox[2] = cpy.relativeBoundingBox[2]; + relativeBoundingBox[3] = cpy.relativeBoundingBox[3]; + } + float relativeBoundingBox[4]; +}; \ No newline at end of file diff --git a/src/ProgressBar.cpp b/src/ProgressBar.cpp index 96ab131..95df0bc 100644 --- a/src/ProgressBar.cpp +++ b/src/ProgressBar.cpp @@ -3,18 +3,21 @@ #include "iostream" ProgressBar::ProgressBar(){ - count = 0; - threshold = 100; + SetThreshold(100); } -int ProgressBar::get_progress(Scene s, char task){ - if(task == *s.result.c_str()){ +int ProgressBar::GetProgress(std::string result, std::string task){ + if(task == result){ count++; } int progress = 100*count/threshold; return progress; } -void ProgressBar::reset_progress(){ +void ProgressBar::ResetProgress(){ count = 0; +} + +void ProgressBar::SetThreshold(int thres) { + threshold = thres; } \ No newline at end of file diff --git a/src/ProgressBar.h b/src/ProgressBar.h index 442a53f..ecd875b 100644 --- a/src/ProgressBar.h +++ b/src/ProgressBar.h @@ -1,11 +1,21 @@ -#include "scene.h" +#ifndef SIGNAPSE_PROGRESSBAR_H +#define SIGNAPSE_PROGRESSBAR_H +#include + +#include "Scene.h" +/*! + * Simple class for keeping track of progress in learning a sign + */ class ProgressBar{ public: ProgressBar(); - int get_progress(Scene s, char task); - void reset_progress(); + int GetProgress(std::string s, std::string task); + void ResetProgress(); + void SetThreshold(int thres); protected: int threshold; - int count; -}; \ No newline at end of file + int count = 0; +}; + +#endif \ No newline at end of file diff --git a/src/qtGenGui.h b/src/QtGeneratedGui.h similarity index 91% rename from src/qtGenGui.h rename to src/QtGeneratedGui.h index 69988fd..40d52c6 100644 --- a/src/qtGenGui.h +++ b/src/QtGeneratedGui.h @@ -24,9 +24,12 @@ #include #include #include +#include QT_BEGIN_NAMESPACE - +/*! + * Auto-generated class, created from parsing our .ui file (gui/TaskMaster.ui) from QtDesigner. As auto-generated elements are not in best code-style, this is wrapped up within the Gui class + */ class Ui_MainWindow { public: @@ -41,6 +44,7 @@ class Ui_MainWindow QGridLayout *gridLayout_2; QSplitter *splitter; QLabel *label_4; + QLabel *label_threshold; QProgressBar *progressBar; QFrame *frame_2; QGridLayout *gridLayout_3; @@ -53,6 +57,7 @@ class Ui_MainWindow QMenuBar *menubar; QMenu *menuMenu; QStatusBar *statusbar; + QSpinBox *spinBox; void setupUi(QMainWindow *MainWindow) { @@ -104,16 +109,35 @@ class Ui_MainWindow splitter->setOrientation(Qt::Horizontal); label_4 = new QLabel(splitter); label_4->setObjectName(QString::fromUtf8("label_4")); + label_4->setAlignment(Qt::AlignRight); splitter->addWidget(label_4); + + progressBar = new QProgressBar(splitter); progressBar->setObjectName(QString::fromUtf8("progressBar")); + QSizePolicy sizePolicy2(QSizePolicy::Expanding, QSizePolicy::Fixed); sizePolicy2.setHorizontalStretch(0); sizePolicy2.setVerticalStretch(2); sizePolicy2.setHeightForWidth(progressBar->sizePolicy().hasHeightForWidth()); progressBar->setSizePolicy(sizePolicy2); - progressBar->setValue(24); + progressBar->setValue(0); splitter->addWidget(progressBar); + + + label_threshold = new QLabel(splitter); + label_threshold->setObjectName(QString::fromUtf8("label_threshold")); + label_threshold->setAlignment(Qt::AlignRight); + splitter->addWidget(label_threshold); + + spinBox = new QSpinBox(splitter); + spinBox->setMaximum(100); + spinBox->setMinimum(1); + spinBox->setValue(100); + spinBox->setObjectName(QString::fromUtf8("spinBox")); + splitter->addWidget(spinBox); + + gridLayout_2->addWidget(splitter, 0, 0, 1, 1); @@ -219,6 +243,7 @@ class Ui_MainWindow label->setText(QString()); label_3->setText(QString()); label_4->setText(QApplication::translate("MainWindow", "Progress:", nullptr)); + label_threshold->setText(QApplication::translate("MainWindow", "Threshold:", nullptr)); const bool __sortingEnabled = listWidget->isSortingEnabled(); listWidget->setSortingEnabled(false); diff --git a/src/Scene.h b/src/Scene.h new file mode 100644 index 0000000..a2fd7a0 --- /dev/null +++ b/src/Scene.h @@ -0,0 +1,55 @@ +#ifndef SIGNAPSE_SCENE_H +#define SIGNAPSE_SCENE_H + + +#include + +//! Struct Point +/*! + * Data structure hold point information for integer points + */ +struct Point +{ + int x, y; + Point() : x(0), y(0) {} + Point(const Point& cpy) : x(cpy.x), y(cpy.y) {} + Point(int x, int y) : x(x), y(y) {} +}; + +//! Struct BoundingBox +/*! + * Data structure holds bounding box point information for a rectangle in image space. + */ +struct BoundingBox{ + Point UpperLeft; + Point LowerRight; + BoundingBox():UpperLeft(), LowerRight(){} + BoundingBox(int upperLeftX, int upperLeftY, int lowerRightX, int lowerRightY): UpperLeft(upperLeftX, upperLeftY), + LowerRight(lowerRightX, lowerRightY) {} + BoundingBox(Point upperLeft, Point lowerRight): UpperLeft(upperLeft), LowerRight(lowerRight) {} +}; + +//! Struct Scene. +/*! + Holds the video frame data and additional Signapse metadata. +*/ +struct Scene{ + Scene(){} + //! Member variable. + /*! + OpenCV n-dimensional dense array object. + */ + cv::Mat frame; + //! Member variable. + /*! + A string representing the predicted output from processing with a CNN model. + */ + std::string result = ""; + //! Member variable + /*! + Region of the frame which frames the user's hand or sign, used to allow cropping for further processing. Variable contains 4 integers, the first two represent the upper-left bounding box coord with the second two representing lower-right. Coordinates are in (column,row) format, units are in pixels. + */ + BoundingBox regionOfInterest; +}; + +#endif \ No newline at end of file diff --git a/src/SceneCallback.h b/src/SceneCallback.h index 2267784..7cbc1d8 100644 --- a/src/SceneCallback.h +++ b/src/SceneCallback.h @@ -1,14 +1,13 @@ -// -// Created by ross on 28/03/2022. -// - #ifndef SIGNAPSE_SCENECALLBACK_H #define SIGNAPSE_SCENECALLBACK_H -#include "scene.h" +#include "Scene.h" +/*! + * A simple callback interface for passing Scenes through the pipeline + */ class SceneCallback{ public: - virtual void nextScene(Scene next) = 0; + virtual void NextScene(Scene next) = 0; }; -#endif //SIGNAPSE_SCENECALLBACK_H +#endif \ No newline at end of file diff --git a/src/SchedulableLink.cpp b/src/SchedulableLink.cpp new file mode 100644 index 0000000..a379838 --- /dev/null +++ b/src/SchedulableLink.cpp @@ -0,0 +1,54 @@ +#include "SchedulableLink.h" +#include "BlockingQueue.cpp" + + +/*! + * Looping process to process scenes and call-back to next pipeline element. Worker thread is slept while no elements are available using BlockingQueue functionality + */ +void SchedulableLink::Run(){ + //waits for scenes to appear on the scheduleQueue, + while(isOn){ + Scene s = scheduleQueue.Pop(); //Blocking Queue Sleeps Execution Until Scene arrives + Scene out = ProcessScene(s); + if(!sceneCallback) continue; + sceneCallback->NextScene(out); + } +} +/*! + * Adds a scene to the intenal scheduleQueue + * @param s + */ +void SchedulableLink::Enqueue(Scene s) { + scheduleQueue.Push(s); +} +/*! + * @return true if the scheduleQueue is empty; false otherwise. + */ +bool SchedulableLink::Available() { + return scheduleQueue.IsEmpty(); //singly threaded for now +} + +/*! + * Starts execution with the worker thread. + */ +void SchedulableLink::Start() { + scheduleWorker = std::thread(&SchedulableLink::Run, this); +} +/*! + * Turns off the link, kills worker thread. Must be called to release thread resources. + */ +void SchedulableLink::Stop(){ + isOn = false; + scheduleWorker.join(); +} +/*! + * If space is available on the scheduleQueue, add the scene to the queue. Otherwise skip. + * @param scene + */ +void SchedulableLink::NextScene(Scene scene) { + //if space on schedule queue, add this scene; otherwise pass the scene through + if(scheduleQueue.IsEmpty()) { //singly threaded for now + scheduleQueue.Push(scene); + } +} + diff --git a/src/SchedulableLink.h b/src/SchedulableLink.h new file mode 100644 index 0000000..d651421 --- /dev/null +++ b/src/SchedulableLink.h @@ -0,0 +1,33 @@ +#ifndef SIGNAPSE_SCHEDULABLESCENELINKER_H +#define SIGNAPSE_SCHEDULABLESCENELINKER_H + +#include +#include +#include + +#include "PipelineLink.h" +#include "BlockingQueue.h" + + +/*! + * A class which extends PipelineLink to handle latency-bound stages. + * The NextScene function is inherited to add Scenes to an internal BlockingQueue. The pure virtual function ProcessScene must be implemented by derived classes to define Scene processing behaviour. The Run method is called in a separate thread and handles waking up the thread when scenes are available and calling back to the next pipeline element. + * BlockingQueue is used as a scheduling mechanism for future extension to multithreading. + */ +class SchedulableLink : public PipelineLink{ +public: + void NextScene(Scene s); + virtual Scene ProcessScene(Scene s) = 0; + void Start(); + void Stop(); + bool Available(); +protected: + void Enqueue(Scene s); + void Run(); + BlockingQueue scheduleQueue; + bool isOn = true; + std::thread scheduleWorker; +}; + + +#endif //SIGNAPSE_SCHEDULABLESCENELINKER_H diff --git a/src/SignapseUtils.h b/src/SignapseUtils.h index 9e8741d..fd2fbae 100644 --- a/src/SignapseUtils.h +++ b/src/SignapseUtils.h @@ -1,49 +1,74 @@ -// -// Created by ross on 28/03/2022. -// - #ifndef SIGNAPSE_SIGNAPSEUTILS_H #define SIGNAPSE_SIGNAPSEUTILS_H + #include #include +#include #include "iostream" #include "SignapseUtils.h" #define MODEL_PATH "models/asl-mobilenetv2.pb" +#define NR_TASKS 26 +namespace { +/*! + * Simple static class for some utilities needed by Signapse objects. + */ + class SignapseUtils { + public: + /*! + * Array of ordered results from expected at the output of the current CNN model in use. + */ + static std::vector results; + + /*! + * Prints welcome message and ASCII art logo + */ + static void welcomeMessage() { + printf(" _____ _ \n" + " / ____(_) \n" + " | (___ _ __ _ _ __ __ _ _ __ ___ ___ \n" + " \\___ \\| |/ _` | '_ \\ / _` | '_ \\/ __|/ _ \\\n" + " ____) | | (_| | | | | (_| | |_) \\__ \\ __/\n" + " |_____/|_|\\__, |_| |_|\\__,_| .__/|___/\\___|\n" + " __/ | | | \n" + " |___/ |_| \n \nWelcome to Signapse, the tool for helping everyday people learn sign language for free!\n"); + } + /*! + * Used to convert an integer index result into a string sign + * @param digit integer index + * @return std::string representing sign at given index + */ + static std::string getLetterFromDigit(int digit) { + return results[digit % results.size()]; + } + /*! + * Generates a random sign for use assigning tasks to users. + * + */ + static std::string makeTask() { + int task_int = rand() % NR_TASKS; + return getLetterFromDigit(task_int); + } + /*! + * Sets the random seed to current system time. + * + */ + static void randSeed(){ + srand((unsigned) time(NULL)); + } + /*! + * @return Path of the currently used CNN network. Path given as std::string + */ + static std::string getModelPath() { + return MODEL_PATH; + } + }; + /*! + * Declares results array, this may be changed for different CNN output results. + */ + std::vector SignapseUtils::results = {"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", + "N", "O", "P", "Q", "R", "S", "T", "U", "V", + "W", "X", "Y", "Z", "del", "space", "nothing"}; +} -class SignapseUtils { -public: - static std::string getLetterFromDigit(int digit){ - std::string results[] = {"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", - "W", "X", "Y", "Z", "del", "space", "nothing"}; - return results[digit]; - } - static void welcomeMessage(){ - printf(" _____ _ \n" - " / ____(_) \n" - " | (___ _ __ _ _ __ __ _ _ __ ___ ___ \n" - " \\___ \\| |/ _` | '_ \\ / _` | '_ \\/ __|/ _ \\\n" - " ____) | | (_| | | | | (_| | |_) \\__ \\ __/\n" - " |_____/|_|\\__, |_| |_|\\__,_| .__/|___/\\___|\n" - " __/ | | | \n" - " |___/ |_| \n \nWelcome to Signapse, the tool for helping everyday people learn sign language for free!"); - } - - - static void printProgress(float percentage, int prediction) { - std::string letter = getLetterFromDigit(prediction); - std::cout << "\r Prediction: " << letter << ", Progress: " << percentage << "% - keep signing! "; - fflush(stdout); - } - - static int makeTask(){ - return rand() % 25; - } - - static std::string getModelPath(){ - return MODEL_PATH; - } -}; - - -#endif //SIGNAPSE_SIGNAPSEUTILS_H +#endif diff --git a/src/camera.cpp b/src/camera.cpp deleted file mode 100644 index a42162d..0000000 --- a/src/camera.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// -// Created by adam on 09/02/2022. -// - -#include "camera.h" -#include "scene.h" -#include -#include -#include -#include -#include "FrameEditor.cpp" - - -Camera::Camera() { - isOn=true; -} - - -void Camera::setOn(bool state){ - isOn = state; -} - -void Camera::threadLoop(){ - while(isOn){ - dataReady(); - } -} - -void Camera::postFrame(SceneCallback* callback){ - if(!callback) return; - cv::Mat temp; - videoCapture.read(temp); - // check if we succeeded - if (temp.empty()) { - std::cerr << "ERROR! blank frame grabbed\n"; - } - cv::Mat frame_bgr; - cv::Mat frame; - cv::flip(temp, frame_bgr, 1); - cv::cvtColor(frame_bgr, frame, cv::COLOR_BGR2RGB); - cv::Size sz = frame.size(); - - Scene s = Scene{ - .frame=frame, - .timestamp = 1, - .result = {}, - .regionOfInterest = {(int)(sz.width * boundingBox[0]), - (int)(sz.height * boundingBox[1]), - (int)(sz.width * boundingBox[2]), - (int)(sz.height * boundingBox[3])} - }; - s = FrameEditor::drawBox(s); - callback->nextScene(s); -} - -void Camera::dataReady(){ - postFrame(cnnCallback); - postFrame(frameCallback); -} - -void Camera::Populate(){ - cv::Mat frame; - videoCapture.read(frame); - // check if we succeeded - if (frame.empty()) { - std::cerr << "ERROR! blank frame grabbed\n"; - } - cv::Size sz = frame.size(); - Scene s = Scene{ - .frame=frame, - .timestamp = 1, - .result = {}, - .regionOfInterest = {(int)(sz.width * boundingBox[0]), - (int)(sz.height * boundingBox[1]), - (int)(sz.width * boundingBox[2]), - (int)(sz.height * boundingBox[3])} - }; - sceneQueue.Push(s); -} - -void Camera::start_thread(){ - videoCapture.open(deviceID, apiID); - cameraThread = std::thread(&Camera::threadLoop, this); -} - -void Camera::Stream() { - while (1) { - if(isOn){ - Populate(); - } - } -} - -float* Camera::getBoundingBox() { - - return boundingBox; -} - -bool Camera::getOn() { - return isOn; -} - -void Camera ::setBoundingBox(float upperLeftX, float upperLeftY, float lowerRightX, float lowerRightY) { - boundingBox[0] = upperLeftX; - boundingBox[1] = upperLeftY; - boundingBox[2] = lowerRightX; - boundingBox[3] = lowerRightY; - for(int i = 0; i < 4; i++){ - float val = boundingBox[i]; - if(val > 1.0f){ - boundingBox[i] = 1.0f; - } - else if (val < 0.0f){ - boundingBox[i] = 0.0f; - } - } -} - -void Camera::registerCNNCallback(SceneCallback *cnncb) { - cnnCallback = cnncb; -} - -void Camera::registerFrameCallback(SceneCallback *fcb) { - frameCallback = fcb; -} \ No newline at end of file diff --git a/src/camera.h b/src/camera.h deleted file mode 100644 index d68e089..0000000 --- a/src/camera.h +++ /dev/null @@ -1,99 +0,0 @@ -// -// Created by Adam on 09/02/2022. -// - -#ifndef SIGNAPSE_CAMERA_H -#define SIGNAPSE_CAMERA_H -#include -#include "reel.h" -#include -#include -#include "SceneCallback.h" - -//! Camera class which inherits from Reel. -/*! - Class for reading web cam data, and creating a Reel of scenes. -*/ -class Camera: public Reel{ -public: - //! Constructor. - /*! - Turns the camera object "ON" and configures the video capture. - */ - Camera(); - bool getOn(); - - void setOn(bool state); - //! Member function. - /*! - Reads webcam data, populates a Scene struct and pushes scene on to sceneQueue. - */ - void Populate(); - //! Member function for starting video capturing thread. - /*! - Starts "Stream" private member function as a thread. - */ - void start_thread(); - //! Member function for setting a new bounding box. - /*! - Updates boundingBox variable with a new set of coordinates. - */ - void setBoundingBox(float upperLeftX, float upperLeftY, float lowerRightX, float lowerRightY); - - float* getBoundingBox(); - - void registerCNNCallback(SceneCallback* cnncb); - - void registerFrameCallback(SceneCallback* fcb); - - - void dataReady(); - - void Start(); - -private: - void postFrame(SceneCallback* callback); - - void threadLoop(); - - //! Private member function. - /*! - Calls "Populate" member function whilst the camera object is "ON" - */ - void Stream(); - //! Private member object. - /*! - OpenCV object for reading web cam data. - */ - cv::VideoCapture videoCapture; - //! Private member variable containing web cam device ID. - /*! - 0 = open default camera - - */ - int deviceID = 0; - //! Private member variable containing web cam API ID. - /*! - 0 = autodetect default API - */ - int apiID = cv::CAP_ANY; - //! Private member thread - std::thread cameraThread; - //! Private member variable containing camera object status. - /*! - false = Camera is off. - true = Camera is on. - */ - bool isOn = false; - //! Private member variable indicating the bounding box to set the region of interest to for each frame. - /*! - Box within the frame which bounds the user's hand or sign, used to allow cropping for further processing. Variable contains 4 floats, the first two represent the upper-left bounding box coord with the second two representing lower-right. Coordinates are in (x,y) format and are defined as a fraction of the total frame width and height in range (0-1). - */ - float boundingBox[4]; - - SceneCallback* cnnCallback = nullptr; - SceneCallback* frameCallback = nullptr; -}; - - -#endif //SIGNAPSE_CAMERA_H diff --git a/src/main.cpp b/src/main.cpp index 6e1f066..73f27bf 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,40 +1,60 @@ #include #include -#include #include + #include #include #include -#include "reel.h" -#include "camera.h" + +#include "Camera.h" #include "CNNProcessor.h" #include "stdlib.h" #include "Gui.h" -#include "FrameEditor.h" +#include "PreProcessor.h" #include "SignapseUtils.h" +#include "LinkSplitter.h" using namespace cv; using namespace std; -//#define THRES 100 - - - - - int main(int argc, char* argv[]){ - QApplication app(argc, argv); SignapseUtils::welcomeMessage(); - Gui gui; - CNNProcessor cnn(SignapseUtils::getModelPath()); - Camera c; - gui.set_task('A'); - c.setBoundingBox(0.25, 0.25, 0.75, 0.75); - c.registerFrameCallback(&gui); - c.registerCNNCallback(&cnn); - cnn.registerCallback(&gui); - c.start_thread(); + + //init Qt and Ui + QApplication app(argc, argv); + QMainWindow window; + Ui_MainWindow ui; + + //make pipeline components + CameraSettings cameraSettings; + Camera camera(cameraSettings); + PreProcessorSettings preProcessorSettings; + PreProcessor preProcessor(preProcessorSettings); + LinkSplitter linkSplitter; + CNNProcessorSettings cnnSettings; + CNNProcessor cnn(cnnSettings); + Gui gui(&window, &ui); + + //register callbacks (link pipeline) + camera.RegisterCallback(&preProcessor); + preProcessor.RegisterCallback(&linkSplitter); + linkSplitter.RegisterCallback(&cnn); + linkSplitter.RegisterSecondaryCallback(&gui); + cnn.RegisterCallback(&gui); + + //start camera and cnn + cnn.Start(); + camera.Start(); + + //start gui gui.SetVisible(true); - app.exec(); + app.exec(); //loops main thread + + //stop camera and cnn + camera.Stop(); + cnn.Stop(); + + //exit + return 0; } diff --git a/src/reel.cpp b/src/reel.cpp deleted file mode 100644 index 384b31a..0000000 --- a/src/reel.cpp +++ /dev/null @@ -1,24 +0,0 @@ -// -// Created by ross on 09/02/2022. -// - -#include "reel.h" -#include "scene.h" -using namespace std; - - -Reel::Reel(){ - -} - -bool Reel::IsEmpty(){ - return sceneQueue.IsEmpty(); -} - -Scene Reel::Pop(){ - return sceneQueue.Pop(); -} - -int Reel::GetNumber(){ - return reelNr; -} diff --git a/src/reel.h b/src/reel.h deleted file mode 100644 index 2df8ff7..0000000 --- a/src/reel.h +++ /dev/null @@ -1,50 +0,0 @@ -// -// Created by ross on 09/02/2022. -// - -#ifndef SIGNAPSE_REEL_H -#define SIGNAPSE_REEL_H -#include "scene.h" -#include "BlockingQueue.h" -#include "BlockingQueue.cpp" - - -//! Reel class. -/*! - Class for interfacing with a queue of Scene -*/ -class Reel{ -public: - //! Constructor. - /*! - No functionality. - */ - Reel(); - //! Member function returns the next Scene in the queue. - /*! - \return elem the next Scene in the queue. - */ - Scene Pop(); - //! Member function returns the Reel number. - /*! - \return reelNr an integer. - */ - int GetNumber(); - bool IsEmpty(); - - -protected: - //! Protected member variable. - /*! - Holds value of the Reel number. - */ - int reelNr; - //! Protected member variable. - /*! - A queue of Scene. - */ - BlockingQueue sceneQueue; -}; - - -#endif //SIGNAPSE_REEL_H diff --git a/src/scene.h b/src/scene.h deleted file mode 100644 index b485d19..0000000 --- a/src/scene.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Created by adam on 09/02/2022. -// -#include -#ifndef SIGNAPSE_SCENE_H -#define SIGNAPSE_SCENE_H - -//! Struct Scene. -/*! - Holds the video frame data and additional Signapse metadata. -*/ -struct Scene{ - //! Member variable. - /*! - OpenCV n-dimensional dense array object. - */ - cv::Mat frame; - //! Member variable. - /*! - Holds timestamp of when the Scene was created by Populate. - */ - int timestamp; - //! Member variable. - /*! - A string representing the predicted output from processing with a CNN model. - */ - std::string result; - //! Member variable - /*! - Region of the frame which frames the user's hand or sign, used to allow cropping for further processing. Variable contains 4 integers, the first two represent the upper-left bounding box coord with the second two representing lower-right. Coordinates are in (column,row) format, units are in pixels. - */ - int regionOfInterest[4]; -}; - -#endif //SIGNAPSE_SCENE_H \ No newline at end of file diff --git a/src/taskMaster.h b/src/taskMaster.h deleted file mode 100644 index 36bab94..0000000 --- a/src/taskMaster.h +++ /dev/null @@ -1,204 +0,0 @@ -/******************************************************************************** -** Form generated from reading UI file 'Taskmaster.ui' -** -** Created by: Qt User Interface Compiler version 5.15.2 -** -** WARNING! All changes made in this file will be lost when recompiling UI file! -********************************************************************************/ - -#ifndef UI_TASKMASTER_H -#define UI_TASKMASTER_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -QT_BEGIN_NAMESPACE - -class Ui_MainWindow -{ -public: - QAction *actionClose; - QAction *actionReload; - QWidget *centralwidget; - QGridLayout *gridLayout; - QFrame *frame_2; - QGridLayout *gridLayout_3; - QPushButton *pushButton; - QFrame *frame_3; - QGridLayout *gridLayout_4; - QListWidget *listWidget; - QFrame *frame_4; - QLabel *label_2; - QFrame *frame; - QLabel *label; - QLabel *label_3; - QMenuBar *menubar; - QMenu *menuMenu; - QStatusBar *statusbar; - - void setupUi(QMainWindow *MainWindow) - { - if (MainWindow->objectName().isEmpty()) - MainWindow->setObjectName(QString::fromUtf8("MainWindow")); - MainWindow->resize(725, 448); - actionClose = new QAction(MainWindow); - actionClose->setObjectName(QString::fromUtf8("actionClose")); - actionReload = new QAction(MainWindow); - actionReload->setObjectName(QString::fromUtf8("actionReload")); - centralwidget = new QWidget(MainWindow); - centralwidget->setObjectName(QString::fromUtf8("centralwidget")); - gridLayout = new QGridLayout(centralwidget); - gridLayout->setObjectName(QString::fromUtf8("gridLayout")); - frame_2 = new QFrame(centralwidget); - frame_2->setObjectName(QString::fromUtf8("frame_2")); - QSizePolicy sizePolicy(QSizePolicy::Preferred, QSizePolicy::Preferred); - sizePolicy.setHorizontalStretch(2); - sizePolicy.setVerticalStretch(0); - sizePolicy.setHeightForWidth(frame_2->sizePolicy().hasHeightForWidth()); - frame_2->setSizePolicy(sizePolicy); - frame_2->setFrameShape(QFrame::StyledPanel); - frame_2->setFrameShadow(QFrame::Raised); - gridLayout_3 = new QGridLayout(frame_2); - gridLayout_3->setObjectName(QString::fromUtf8("gridLayout_3")); - pushButton = new QPushButton(frame_2); - pushButton->setObjectName(QString::fromUtf8("pushButton")); - - gridLayout_3->addWidget(pushButton, 2, 0, 1, 1); - - frame_3 = new QFrame(frame_2); - frame_3->setObjectName(QString::fromUtf8("frame_3")); - QSizePolicy sizePolicy1(QSizePolicy::Preferred, QSizePolicy::Preferred); - sizePolicy1.setHorizontalStretch(0); - sizePolicy1.setVerticalStretch(40); - sizePolicy1.setHeightForWidth(frame_3->sizePolicy().hasHeightForWidth()); - frame_3->setSizePolicy(sizePolicy1); - frame_3->setFrameShape(QFrame::StyledPanel); - frame_3->setFrameShadow(QFrame::Raised); - gridLayout_4 = new QGridLayout(frame_3); - gridLayout_4->setObjectName(QString::fromUtf8("gridLayout_4")); - listWidget = new QListWidget(frame_3); - QFont font; - font.setFamily(QString::fromUtf8("Helvetica")); - QListWidgetItem *__qlistwidgetitem = new QListWidgetItem(listWidget); - __qlistwidgetitem->setTextAlignment(Qt::AlignCenter); - __qlistwidgetitem->setFont(font); - new QListWidgetItem(listWidget); - QFont font1; - font1.setFamily(QString::fromUtf8("Helvetica")); - font1.setPointSize(48); - QListWidgetItem *__qlistwidgetitem1 = new QListWidgetItem(listWidget); - __qlistwidgetitem1->setTextAlignment(Qt::AlignCenter); - __qlistwidgetitem1->setFont(font1); - listWidget->setObjectName(QString::fromUtf8("listWidget")); - - gridLayout_4->addWidget(listWidget, 0, 0, 1, 1); - - - gridLayout_3->addWidget(frame_3, 0, 0, 1, 1); - - frame_4 = new QFrame(frame_2); - frame_4->setObjectName(QString::fromUtf8("frame_4")); - QSizePolicy sizePolicy2(QSizePolicy::Preferred, QSizePolicy::Preferred); - sizePolicy2.setHorizontalStretch(0); - sizePolicy2.setVerticalStretch(60); - sizePolicy2.setHeightForWidth(frame_4->sizePolicy().hasHeightForWidth()); - frame_4->setSizePolicy(sizePolicy2); - frame_4->setFrameShape(QFrame::StyledPanel); - frame_4->setFrameShadow(QFrame::Raised); - label_2 = new QLabel(frame_4); - label_2->setObjectName(QString::fromUtf8("label_2")); - label_2->setGeometry(QRect(10, 10, 151, 141)); - QSizePolicy sizePolicy3(QSizePolicy::Preferred, QSizePolicy::Preferred); - sizePolicy3.setHorizontalStretch(3); - sizePolicy3.setVerticalStretch(0); - sizePolicy3.setHeightForWidth(label_2->sizePolicy().hasHeightForWidth()); - label_2->setSizePolicy(sizePolicy3); - label_2->setFrameShape(QFrame::WinPanel); - - gridLayout_3->addWidget(frame_4, 1, 0, 1, 1); - - - gridLayout->addWidget(frame_2, 0, 1, 1, 1); - - frame = new QFrame(centralwidget); - frame->setObjectName(QString::fromUtf8("frame")); - QSizePolicy sizePolicy4(QSizePolicy::Preferred, QSizePolicy::Preferred); - sizePolicy4.setHorizontalStretch(5); - sizePolicy4.setVerticalStretch(0); - sizePolicy4.setHeightForWidth(frame->sizePolicy().hasHeightForWidth()); - frame->setSizePolicy(sizePolicy4); - frame->setFrameShape(QFrame::StyledPanel); - frame->setFrameShadow(QFrame::Raised); - label = new QLabel(frame); - label->setObjectName(QString::fromUtf8("label")); - label->setGeometry(QRect(30, 40, 401, 321)); - label->setFrameShape(QFrame::WinPanel); - label_3 = new QLabel(frame); - label_3->setObjectName(QString::fromUtf8("label_3")); - label_3->setGeometry(QRect(-60, -10, 211, 151)); - label_3->setFrameShape(QFrame::NoFrame); - label_3->setPixmap(QPixmap(QString::fromUtf8(":/logo/logoResize.png"))); - - gridLayout->addWidget(frame, 0, 0, 1, 1); - - MainWindow->setCentralWidget(centralwidget); - menubar = new QMenuBar(MainWindow); - menubar->setObjectName(QString::fromUtf8("menubar")); - menubar->setGeometry(QRect(0, 0, 725, 24)); - menuMenu = new QMenu(menubar); - menuMenu->setObjectName(QString::fromUtf8("menuMenu")); - MainWindow->setMenuBar(menubar); - statusbar = new QStatusBar(MainWindow); - statusbar->setObjectName(QString::fromUtf8("statusbar")); - MainWindow->setStatusBar(statusbar); - - menubar->addAction(menuMenu->menuAction()); - menuMenu->addAction(actionClose); - menuMenu->addAction(actionReload); - - retranslateUi(MainWindow); - - QMetaObject::connectSlotsByName(MainWindow); - } // setupUi - - void retranslateUi(QMainWindow *MainWindow) - { - MainWindow->setWindowTitle(QCoreApplication::translate("MainWindow", "MainWindow", nullptr)); - actionClose->setText(QCoreApplication::translate("MainWindow", "Close", nullptr)); - actionReload->setText(QCoreApplication::translate("MainWindow", "Reload", nullptr)); - pushButton->setText(QCoreApplication::translate("MainWindow", "Next Task", nullptr)); - - const bool __sortingEnabled = listWidget->isSortingEnabled(); - listWidget->setSortingEnabled(false); - QListWidgetItem *___qlistwidgetitem = listWidget->item(0); - ___qlistwidgetitem->setText(QCoreApplication::translate("MainWindow", "Task:", nullptr)); - QListWidgetItem *___qlistwidgetitem1 = listWidget->item(2); - ___qlistwidgetitem1->setText(QCoreApplication::translate("MainWindow", "C", nullptr)); - listWidget->setSortingEnabled(__sortingEnabled); - - label_2->setText(QString()); - label->setText(QString()); - label_3->setText(QString()); - menuMenu->setTitle(QCoreApplication::translate("MainWindow", "Menu", nullptr)); - } // retranslateUi - -}; - -namespace Ui { - class MainWindow: public Ui_MainWindow {}; -} // namespace Ui - -QT_END_NAMESPACE - -#endif // UI_TASKMASTER_H \ No newline at end of file diff --git a/src/taskmaster.cpp b/src/taskmaster.cpp deleted file mode 100644 index 27037c1..0000000 --- a/src/taskmaster.cpp +++ /dev/null @@ -1,12 +0,0 @@ -// -// Created by adam on 09/02/2022. -// - -#include "taskmaster.h" - -Taskmaster::Taskmaster(){ - -} -void Taskmaster::set_task(char new_task){ - master_task = new_task; -} \ No newline at end of file diff --git a/src/taskmaster.h b/src/taskmaster.h deleted file mode 100644 index 7b5ee6a..0000000 --- a/src/taskmaster.h +++ /dev/null @@ -1,32 +0,0 @@ -// -// Created by adam on 09/02/2022. -// - -#ifndef SIGNAPSE_TASKMASTER_H -#define SIGNAPSE_TASKMASTER_H -//! Taskmaster class. -/*! - Class containing member functions for setting the global signing task. -*/ -class Taskmaster{ -public: - //! Constructor. - /*! - No functionality. - */ - Taskmaster(); - //! Public member function takes new task as argument and assigns value to the current task. - /*! - \param new_task char containing the new task. - */ - void set_task(char new_task); -protected: - //! Protected member variable. - /*! - Holds value of the master/global task. - */ - char master_task; -}; - - -#endif //SIGNAPSE_TASKMASTER_H diff --git a/test.sh b/test.sh index b62534b..e87b68d 100644 --- a/test.sh +++ b/test.sh @@ -1,2 +1 @@ -cd test -./main \ No newline at end of file +./test/main diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bb9e0dd..59d8810 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,6 +1,5 @@ cmake_minimum_required(VERSION 3.0) project(Signapse) - set(OpenCV_DIR "../opencv_build") find_package(OpenCV REQUIRED) find_package(Qt5Widgets REQUIRED) @@ -11,4 +10,5 @@ include_directories(../src) enable_testing() add_executable(main main.cpp) -target_link_libraries(main ${OpenCV_LIBS} Qt5::Widgets gtest gtest_main camera CNNProcessor) +target_link_libraries(main ${OpenCV_LIBS} Qt5::Widgets gtest gtest_main Camera CNNProcessor BlockingQueue PreProcessor) + diff --git a/test/camera_test.cpp b/test/camera_test.cpp deleted file mode 100644 index 4315ca8..0000000 --- a/test/camera_test.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main(){ - Camera test_camera; - return 0; -} \ No newline at end of file diff --git a/test/gtest_blockingqueue.h b/test/gtest_blockingqueue.h new file mode 100644 index 0000000..48e064f --- /dev/null +++ b/test/gtest_blockingqueue.h @@ -0,0 +1,28 @@ +#ifndef SIGNAPSE_GTEST_BLOCKINGQUEUE_H +#define SIGNAPSE_GTEST_BLOCKINGQUEUE_H + +#include "BlockingQueue.h" +#include "BlockingQueue.cpp" + +TEST(blockingqueue_test, checkInit_empty){ + BlockingQueue queue; + EXPECT_EQ(queue.Size(), 0); +} + +TEST(blockingqueue_test, checkPush_size){ + BlockingQueue queue; + for(int i = 1; i<11; i++){ + queue.Push(i); + EXPECT_EQ(queue.Size(), i); + } +} + +TEST(blockingqueue_test, checkIntegrity_int){ + BlockingQueue queue; + int toQueue = 314159; + queue.Push(toQueue); + int popped = queue.Pop(); + EXPECT_EQ(toQueue, popped); +} + +#endif //SIGNAPSE_GTEST_BLOCKINGQUEUE_H diff --git a/test/gtest_camera.h b/test/gtest_camera.h index ed61347..f14db01 100644 --- a/test/gtest_camera.h +++ b/test/gtest_camera.h @@ -1,5 +1,5 @@ #include -#include "camera.h" +#include "Camera.h" TEST(camera_test, checkIsOn_init){ @@ -15,24 +15,3 @@ TEST(camera_test, checkIsOn_set){ EXPECT_EQ(c.getOn(), true); } -TEST(camera_test, checkBoundingBox_set_values){ - Camera c; - c.setBoundingBox(0.0,0.0,0.0,0.0); - float* bbox = c.getBoundingBox(); - for(int i = 0; i < 4; i++){ - EXPECT_EQ(bbox[i], 0.0f); - } -} -TEST(camera_test, checkBoundingBox_set_limits){ - Camera c; - c.setBoundingBox(-1.0,-1.0,-1.0,-1.0); - float* bbox = c.getBoundingBox(); - for(int i = 0; i < 4; i++){ - EXPECT_EQ(bbox[i], 0.0f); - } - c.setBoundingBox(2.0,2.0,2.0,2.0); - bbox = c.getBoundingBox(); - for(int i = 0; i < 4; i++){ - EXPECT_EQ(bbox[i], 1.0f); - } -} \ No newline at end of file diff --git a/test/gtest_cnnprocessor.h b/test/gtest_cnnprocessor.h index a432009..bc759f4 100644 --- a/test/gtest_cnnprocessor.h +++ b/test/gtest_cnnprocessor.h @@ -1,24 +1,13 @@ #include -#include -#include -#include +#include "gtest_tools.h" + + #include "CNNProcessor.h" #include "SignapseUtils.h" -Scene MakeScene(std::string letter){ - std::string filepath = "asl_alphabet_test/" + letter + "_test.jpg"; - cv::Mat test_frame = cv::imread(filepath); - cv::Mat flipped; cv::flip(test_frame, flipped, 1); - Scene test_scene; - test_scene.regionOfInterest[0] = 0; - test_scene.regionOfInterest[1] = 0; - test_scene.regionOfInterest[2] = flipped.rows; - test_scene.regionOfInterest[3] = flipped.cols; - test_scene.frame = flipped; - return test_scene; -} TEST(cnnprocessor_test, checkMakeBlob_dims){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene s = MakeScene("A"); cv::Mat frame = s.frame; cv::Mat blob = cnn.MakeBlob(s); @@ -26,140 +15,158 @@ TEST(cnnprocessor_test, checkMakeBlob_dims){ } TEST(cnnprocessor_test, checkResult_a){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("A")); EXPECT_EQ(out.result, "A"); } TEST(cnnprocessor_test, checkResult_b){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("B")); EXPECT_EQ(out.result, "B"); } TEST(cnnprocessor_test, checkResult_c){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("C")); EXPECT_EQ(out.result, "C"); } TEST(cnnprocessor_test, checkResult_d){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("D")); EXPECT_EQ(out.result, "D"); } TEST(cnnprocessor_test, checkResult_e){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("E")); EXPECT_EQ(out.result, "E"); } TEST(cnnprocessor_test, checkResult_f){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("F")); EXPECT_EQ(out.result, "F"); } TEST(cnnprocessor_test, checkResult_g){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("G")); EXPECT_EQ(out.result, "G"); } TEST(cnnprocessor_test, checkResult_h){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("H")); EXPECT_EQ(out.result, "H"); } TEST(cnnprocessor_test, checkResult_j){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("J")); EXPECT_EQ(out.result, "J"); } -TEST(cnnprocessor_test, checkResult_k){ - CNNProcessor cnn(SignapseUtils::getModelPath()); - Scene out = cnn.ProcessScene(MakeScene("K")); - EXPECT_EQ(out.result, "K"); -} TEST(cnnprocessor_test, checkResult_l){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("L")); EXPECT_EQ(out.result, "L"); } TEST(cnnprocessor_test, checkResult_m){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("M")); EXPECT_EQ(out.result, "M"); } TEST(cnnprocessor_test, checkResult_n){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("N")); EXPECT_EQ(out.result, "N"); } TEST(cnnprocessor_test, checkResult_o){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("O")); EXPECT_EQ(out.result, "O"); } TEST(cnnprocessor_test, checkResult_p){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("P")); EXPECT_EQ(out.result, "P"); } TEST(cnnprocessor_test, checkResult_q){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("Q")); EXPECT_EQ(out.result, "Q"); } TEST(cnnprocessor_test, checkResult_r){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("R")); EXPECT_EQ(out.result, "R"); } TEST(cnnprocessor_test, checkResult_s){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("S")); EXPECT_EQ(out.result, "S"); } TEST(cnnprocessor_test, checkResult_t){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("T")); EXPECT_EQ(out.result, "T"); } TEST(cnnprocessor_test, checkResult_u){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("U")); EXPECT_EQ(out.result, "U"); } TEST(cnnprocessor_test, checkResult_w){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("W")); EXPECT_EQ(out.result, "W"); } TEST(cnnprocessor_test, checkResult_x){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("X")); EXPECT_EQ(out.result, "X"); } TEST(cnnprocessor_test, checkResult_y){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("Y")); EXPECT_EQ(out.result, "Y"); } TEST(cnnprocessor_test, checkResult_z){ - CNNProcessor cnn(SignapseUtils::getModelPath()); + CNNProcessorSettings settings; + CNNProcessor cnn(settings); Scene out = cnn.ProcessScene(MakeScene("Z")); EXPECT_EQ(out.result, "Z"); } diff --git a/test/gtest_preprocessor.h b/test/gtest_preprocessor.h new file mode 100644 index 0000000..05bb782 --- /dev/null +++ b/test/gtest_preprocessor.h @@ -0,0 +1,29 @@ +#ifndef SIGNAPSE_GTEST_PREPROCESSOR_H +#define SIGNAPSE_GTEST_PREPROCESSOR_H +#include "gtest_tools.h" + +#include "PreProcessor.h" + + +TEST(preprocessor_test, checkBBox_set){ + EndpointTester endpoint; + PreProcessorSettings settings; + PreProcessor pp(settings); + pp.SetBoundingBox(0.0f, 0.0f, 1.0f, 1.0f); + Scene s = MakeScene("A"); + pp.RegisterCallback(&endpoint); + pp.NextScene(s); + EXPECT_EQ(endpoint.currentScene.regionOfInterest.UpperLeft.x, 0); + EXPECT_EQ(endpoint.currentScene.regionOfInterest.UpperLeft.y, 0); + EXPECT_EQ(endpoint.currentScene.regionOfInterest.LowerRight.x, endpoint.currentScene.frame.rows); + EXPECT_EQ(endpoint.currentScene.regionOfInterest.LowerRight.y, endpoint.currentScene.frame.cols); +} + + + + + + + + +#endif //SIGNAPSE_GTEST_PREPROCESSOR_H diff --git a/test/gtest_scene.h b/test/gtest_scene.h new file mode 100644 index 0000000..2a167ea --- /dev/null +++ b/test/gtest_scene.h @@ -0,0 +1,25 @@ +#ifndef SIGNAPSE_GTEST_SCENE_H +#define SIGNAPSE_GTEST_SCENE_H +#include + +#include "Scene.h" + +TEST(test_scene, checkinit_frame){ + Scene s; + EXPECT_TRUE(s.frame.empty()); +} + +TEST(test_scene, checkinit_result){ + Scene s; + EXPECT_EQ(s.result, ""); +} + +TEST(test_scene, checkinit_roi){ + Scene s; + EXPECT_EQ(s.regionOfInterest.UpperLeft.x, 0); + EXPECT_EQ(s.regionOfInterest.UpperLeft.y, 0); + EXPECT_EQ(s.regionOfInterest.LowerRight.x, 0); + EXPECT_EQ(s.regionOfInterest.LowerRight.y, 0); +} + +#endif //SIGNAPSE_GTEST_SCENE_H diff --git a/test/gtest_signapseutils.h b/test/gtest_signapseutils.h index 6e5fece..6771f72 100644 --- a/test/gtest_signapseutils.h +++ b/test/gtest_signapseutils.h @@ -1,3 +1,13 @@ #include #include "SignapseUtils.h" +TEST(signapseutils_test, checkModelPath_ret){ + std::string modelPath = SignapseUtils::getModelPath(); + EXPECT_EQ(modelPath, MODEL_PATH); +} + +TEST(signapseutils_test, checkMakeTask_int){ + int size = SignapseUtils::results.size(); + EXPECT_LE(NR_TASKS, size); +} + diff --git a/test/gtest_tools.h b/test/gtest_tools.h new file mode 100644 index 0000000..c2022b9 --- /dev/null +++ b/test/gtest_tools.h @@ -0,0 +1,33 @@ +#ifndef SIGNAPSE_GTEST_TOOLS_H +#define SIGNAPSE_GTEST_TOOLS_H + +#include +#include +#include + +#include "PipelineLink.h" + +class EndpointTester : public PipelineLink{ +public: + void NextScene(Scene s){ + currentScene = s; + } + Scene currentScene; +}; + +Scene MakeScene(std::string letter){ + std::string filepath = "test/asl_alphabet_test/" + letter + "_test.jpg"; + cv::Mat test_frame = cv::imread(filepath); + cv::Mat flipped; cv::flip(test_frame, flipped, 1); + Scene test_scene; + test_scene.regionOfInterest.UpperLeft.x = 0; + test_scene.regionOfInterest.UpperLeft.y = 0; + test_scene.regionOfInterest.LowerRight.x = flipped.rows; + test_scene.regionOfInterest.LowerRight.y = flipped.cols; + test_scene.frame = flipped; + return test_scene; +} + + + +#endif //SIGNAPSE_GTEST_TOOLS_H diff --git a/test/inference_test.cpp b/test/inference_test.cpp deleted file mode 100644 index 4fa638a..0000000 --- a/test/inference_test.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include "SignapseUtils.h" - -int main(){ - - Camera c; - CNNProcessor cnn = CNNProcessor(&c, "../models/asl-mobilenetv2.pb"); - for(int i=0; i<26; i++) - { - char filepath[100]; - sprintf(filepath, "../test/asl_alphabet_test/%c_test.jpg", i+65); - cv::Mat test_frame = cv::imread(filepath); - //printf("rows=%i, cols=%i", test_frame.rows,test_frame.cols); - Scene test_scene; - test_scene.regionOfInterest[0] = 0; - test_scene.regionOfInterest[1] = 0; - test_scene.regionOfInterest[2] = 200; - test_scene.regionOfInterest[3] = 200; - cv::Mat flip_frame; - cv::flip(test_frame, flip_frame, 1); - test_scene.frame = flip_frame; - test_scene = cnn.ProcessScene(test_scene); - std::string result = test_scene.result; - std::string ground_truth_letter = SignapseUtils::getLetterFromDigit(i); - if(result != ground_truth_letter) - { - if(i == 21) break; - //printf("image=%c result=%s\n\r", i+65, result); - return 1; - } - } - return 0; - -} \ No newline at end of file diff --git a/test/main.cpp b/test/main.cpp index c0366cc..4428166 100644 --- a/test/main.cpp +++ b/test/main.cpp @@ -1,6 +1,11 @@ #include "gtest/gtest.h" +#include "gtest_scene.h" #include "gtest_camera.h" #include "gtest_cnnprocessor.h" +#include "gtest_blockingqueue.h" +#include "gtest_preprocessor.h" +#include "gtest_signapseutils.h" +//#include "" int main(int argc, char **argv) {