diff --git a/.gitignore b/.gitignore index 0d4a918..8850456 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ # FileNest .gitignore +#mY STUFF TO IGNORE +tasks\mnist_classifier\src\ray_tune.py + # Environment variables .env .env.local @@ -273,7 +276,7 @@ uploads/ downloads/ cache/ embeddings/ -models/ +# models/ vectors/ # FileNest configuration files (sensitive) diff --git a/README.md b/README.md index 4f3fab6..aa8b69b 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ FileNest/ ```bash # Clone the repository -git clone https://github.com/lakshyajain-0291/FileNest.git +git clone https://github.com/AISocietyIITJ/FileNest.git cd FileNest # Set up backend dependencies diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..56fcae3 --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module filenest + +go 1.24.4 + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.5 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/sync v0.15.0 // indirect + golang.org/x/text v0.26.0 // indirect + gorm.io/driver/postgres v1.6.0 // indirect + gorm.io/gorm v1.30.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7840e65 --- /dev/null +++ b/go.sum @@ -0,0 +1,29 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= +gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= +gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= +gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= diff --git a/tasks/backend_prototype/README.md b/tasks/backend_prototype/README.md new file mode 100644 index 0000000..2f0948e --- /dev/null +++ b/tasks/backend_prototype/README.md @@ -0,0 +1,78 @@ +# FileNest Backend Prototype + +This project is a backend prototype for indexing and storing file embeddings using Go, PostgreSQL (with GORM), and a worker pool architecture. It demonstrates concurrent file processing, embedding generation, and database storage. + +## Features + +- Walks a directory and processes all `.txt` files +- Generates random embeddings for each file (placeholder for real embedding models) +- Calculates cosine similarity to a set of reference embeddings (D1TV) +- Stores file metadata and embeddings in PostgreSQL using GORM +- Uses a worker pool for concurrent file processing +- Graceful shutdown on interrupt + +## Project Structure + +``` +backend_prototype/ +├── db/ +│ └── database.go # Database connection and initialization +├── embedding.go # Embedding and similarity functions +├── go.mod # Go module definition +├── go.sum # Go dependencies +├── main.go # Application entry point +├── models/ +│ └── model.go # FileIndex model definition +├── process.go # File processing logic +├── worker.go # Worker pool and job processing +├── sample_texts/ +│ ├── a.txt ... j.txt # Sample text files for indexing +└── README.md # Project documentation +``` + +## Prerequisites + +- Go 1.21 or higher +- PostgreSQL database +- Set the `POSTGRES_DSN` environment variable with your PostgreSQL connection string + +## Getting Started + +1. **Install dependencies:** + ```sh + go mod tidy + ``` + +2. **Set up your PostgreSQL DSN:** + ```sh + export POSTGRES_DSN="host=localhost user=youruser password=yourpass dbname=yourdb port=5432 sslmode=disable" + ``` + +3. **Run the application:** + ```sh + go run main.go worker.go process.go embedding.go ./sample_texts + ``` + + By default, it will process all `.txt` files in `sample_texts/` using 5 workers. + +4. **Command-line options:** + - `-w`: Number of concurrent workers (default: 5) + - `-dir`: Directory to index (default: ./sample_texts) + - `-timeout`: Per-file processing timeout (default: 5s) + + Example: + ```sh + go run main.go worker.go process.go embedding.go db/database.go models/model.go -w 10 -dir ./sample_texts -timeout 10s + ``` + +## How It Works + +- The app walks through the specified directory, sending `.txt` files to a pool of workers. +- Each worker reads the file, generates a random embedding, finds the most similar D1TV embedding, and stores the result in PostgreSQL. +- Embeddings are stored as arrays using the `pq.Float64Array` type. + +## Notes + +- The embedding generation is a placeholder; replace `generateEmbedding` in `embedding.go` with your actual model. +- The database schema is auto-migrated on startup. +- Sample text files are provided in `sample_texts/`. \ No newline at end of file diff --git a/tasks/backend_prototype/db/database.go b/tasks/backend_prototype/db/database.go new file mode 100644 index 0000000..fd2e06a --- /dev/null +++ b/tasks/backend_prototype/db/database.go @@ -0,0 +1,23 @@ +package db + +import ( + "log" + "os" //to read environment variables + + "backend_prototype/models" + + "gorm.io/driver/postgres" + "gorm.io/gorm" //main GORM library +) + +var DB *gorm.DB //pointer to GORM database connection + +func InitDB() { + dsn := os.Getenv("POSTGRES_DSN") //dsn (Data Source Name) connection string to PostgreSQL, set as environment variable + var err error + DB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{}) //here, postgres.open creates a gorm drive for postgres and gorm.open establishes the actual connection + if err != nil { + log.Fatalf("Failed to connect DB: %v", err) + } + DB.AutoMigrate(&models.FileIndex{}) //telling gorm to update the database schema to match the FileIndex model +} diff --git a/tasks/backend_prototype/embedding.go b/tasks/backend_prototype/embedding.go new file mode 100644 index 0000000..1ed7b08 --- /dev/null +++ b/tasks/backend_prototype/embedding.go @@ -0,0 +1,26 @@ +package main + +import ( + "math" + "math/rand" +) + +const embeddingDim = 128 + +func generateEmbedding(content string) []float64 { //generateEmbedding creates a random embedding vector for the given content, whereas in rela application, we will use a pre-trained model to generate the embedding + vec := make([]float64, embeddingDim) + for i := range vec { + vec[i] = rand.Float64() + } + return vec +} + +func cosineSimilarity(a, b []float64) float64 { + var dot, normA, normB float64 + for i := 0; i < embeddingDim; i++ { + dot += a[i] * b[i] //sum of all dot products + normA += a[i] * a[i] //sum of all square of first vector + normB += b[i] * b[i] + } + return dot / (math.Sqrt(normA) * math.Sqrt(normB)) //cosine similarity s +} diff --git a/tasks/backend_prototype/go.mod b/tasks/backend_prototype/go.mod new file mode 100644 index 0000000..db79362 --- /dev/null +++ b/tasks/backend_prototype/go.mod @@ -0,0 +1,21 @@ +module backend_prototype + +go 1.24.4 + +require ( + github.com/lib/pq v1.10.9 + gorm.io/driver/postgres v1.6.0 + gorm.io/gorm v1.30.0 +) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.6.0 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/text v0.21.0 // indirect +) diff --git a/tasks/backend_prototype/go.sum b/tasks/backend_prototype/go.sum new file mode 100644 index 0000000..428c28c --- /dev/null +++ b/tasks/backend_prototype/go.sum @@ -0,0 +1,38 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= +gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= +gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= +gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= diff --git a/tasks/backend_prototype/main.go b/tasks/backend_prototype/main.go new file mode 100644 index 0000000..da70953 --- /dev/null +++ b/tasks/backend_prototype/main.go @@ -0,0 +1,72 @@ +package main + +import ( + "backend_prototype/db" //custom package for database operations (InitDB) + "context" + "flag" //for command-line flags like -w, -dir, -timeout + "log" + "os" //for file operations and environment variables (setting up postgres data source name (DSN)) + "os/signal" //for ctrl+c handling + "path/filepath" //for walking the directory structure + "sync" //for managing concurrent workers + "time" +) + +func main() { + //all command line flags + numWorkers := flag.Int("w", 5, "number of concurrent workers") //number of concurrent workers, default is 5 + dirPath := flag.String("dir", "./sample_texts", "directory to index") //path to directory to index, default is "./sample_texts" + timeout := flag.Duration("timeout", 5*time.Second, "per-file processing timeout") //timeout for processing each file, default is 5 seconds + + flag.Parse() //reading the flags the cient has set + db.InitDB() //initDB from database.go and gets the model fileindex from models/model.go + + // generating D1TV embeddings + var d1tvs [][]float64 //multidimensional slice to hold D1TV embeddings, can help in automatically increasing the size of the slice as needed + for i := 0; i < 10; i++ { + d1tvs = append(d1tvs, generateEmbedding("D1TV")) //generateEmbedding is a function from worker.go that generates a D1TV embedding for each file + } + + jobs := make(chan FileJob) //chanel to carry file jobs to workers + + ctx, stop := context.WithCancel(context.Background()) //ctx here is a context that is cancelled so as to stop workers at ctrl + c + defer stop() //using stop function as soon as main function exits, here stop is like a variable holding cancel function + + // handling ctrl + c to shutdown properly + go func() { //startinga goroutine + c := make(chan os.Signal, 1) //made a channel to wait for an os signal and it can hold 1 + signal.Notify(c, os.Interrupt) //our program gets notified when we get an os.interrupt + <-c //until someone writes into c channel + log.Println("Interrupt received, shutting down...") + stop() + }() + + var wg sync.WaitGroup //creating a pool of workers using sync.WaitGroup + for i := 1; i <= *numWorkers; i++ { //from 1 to number of workers specified using the -w flag + wg.Add(1) //adding a worker + go func(id int) { //here we are defining the function, calling it and launching it as a goroutine + defer wg.Done() //later tell when the worker is done + startWorker(id, jobs, d1tvs, *timeout) //startWorker is a function from worker.go that does the actual work of processing files + }(i) //calling the function with id as i + } + + // Walk the directory and send file paths to jobs channel + err := filepath.Walk(*dirPath, func(path string, info os.FileInfo, err error) error { //starting to walk the directory specified by -dir flag, calling the function helps us recieve the string path, its metadata or any error it is facing + select { //this select statement is used to handle the context cancellation + case <-ctx.Done(): + return ctx.Err() // + default: //for when context is not cancelled + if err == nil && !info.IsDir() && filepath.Ext(path) == ".txt" { //ensures no error, the file is not soem kind of directory and filters for .txt file + jobs <- FileJob{Path: path} //send the file path to jobs channel + } + return nil //return to show that no error is there + } + }) + if err != nil && err != context.Canceled { //if an error occurs and its not due to cancellation context + log.Fatalf("Walk error: %v", err) + } + + close(jobs) //close the jobs channel to tell that all files have been sent + wg.Wait() //waiting for workers to get processed + log.Println("Indexing complete.") +} diff --git a/tasks/backend_prototype/models/model.go b/tasks/backend_prototype/models/model.go new file mode 100644 index 0000000..fa7fc56 --- /dev/null +++ b/tasks/backend_prototype/models/model.go @@ -0,0 +1,16 @@ +package models + +import ( + "time" + + "github.com/lib/pq" //provides pq.Float64Array for []float64 type +) + +type FileIndex struct { + ID uint `gorm:"primaryKey"` // telling gorm that this is the primary key + Filename string + Filepath string + Embedding pq.Float64Array `gorm:"type:float8[]"` // using pq.Float64Array for []float64 type because gorm does not support []float64 directly + D1TVID int + IndexedAt time.Time +} diff --git a/tasks/backend_prototype/process.go b/tasks/backend_prototype/process.go new file mode 100644 index 0000000..7fa15b9 --- /dev/null +++ b/tasks/backend_prototype/process.go @@ -0,0 +1,45 @@ +package main + +import ( + "backend_prototype/db" + "backend_prototype/models" // for importing the FileIndex struct + "context" + "log" + "os" //for reading files + "path/filepath" + "time" +) + +func processFile(ctx context.Context, workerID int, path string, d1tvs [][]float64) error { //mainly controllin cancellation and timeout of the worker and returning error if any + select { //basic checking to see if the context is done, if so return the error + case <-ctx.Done(): + return ctx.Err() + default: + content, err := os.ReadFile(path) //read the file as bytes + if err != nil { + return err + } + embedding := generateEmbedding(string(content)) //generate the embedding from the file content, this is a placeholder function + + bestID := 0 + bestSim := -1.0 //start with lowest similarity score + for i, vec := range d1tvs { //looping through the D1TV embeddings to find the best match + sim := cosineSimilarity(embedding, vec) + if sim > bestSim { + bestSim = sim + bestID = i + } + } + + db.DB.Create(&models.FileIndex{ //creating fileindex in the databse and .models is here cuz we use package models in model.go + Filename: filepath.Base(path), + Filepath: path, + Embedding: embedding, + D1TVID: bestID, + IndexedAt: time.Now(), + }) + + log.Printf("[Worker-%d] Processed %s → D1TV: %d (similarity: %.2f)", workerID, filepath.Base(path), bestID, bestSim) + return nil + } +} diff --git a/tasks/backend_prototype/sample_texts/a.txt b/tasks/backend_prototype/sample_texts/a.txt new file mode 100644 index 0000000..a38db62 --- /dev/null +++ b/tasks/backend_prototype/sample_texts/a.txt @@ -0,0 +1 @@ +Assassin Creeds \ No newline at end of file diff --git a/tasks/backend_prototype/sample_texts/b.txt b/tasks/backend_prototype/sample_texts/b.txt new file mode 100644 index 0000000..036b4e6 --- /dev/null +++ b/tasks/backend_prototype/sample_texts/b.txt @@ -0,0 +1 @@ +Baldur's Gate 3 \ No newline at end of file diff --git a/tasks/backend_prototype/sample_texts/c.txt b/tasks/backend_prototype/sample_texts/c.txt new file mode 100644 index 0000000..7ce05eb --- /dev/null +++ b/tasks/backend_prototype/sample_texts/c.txt @@ -0,0 +1 @@ +Cyberpunk \ No newline at end of file diff --git a/tasks/backend_prototype/sample_texts/d.txt b/tasks/backend_prototype/sample_texts/d.txt new file mode 100644 index 0000000..dbeb9a0 --- /dev/null +++ b/tasks/backend_prototype/sample_texts/d.txt @@ -0,0 +1 @@ +Demon Souls \ No newline at end of file diff --git a/tasks/backend_prototype/sample_texts/e.txt b/tasks/backend_prototype/sample_texts/e.txt new file mode 100644 index 0000000..55b8800 --- /dev/null +++ b/tasks/backend_prototype/sample_texts/e.txt @@ -0,0 +1 @@ +Elden Ring \ No newline at end of file diff --git a/tasks/backend_prototype/sample_texts/f.txt b/tasks/backend_prototype/sample_texts/f.txt new file mode 100644 index 0000000..b4d3218 --- /dev/null +++ b/tasks/backend_prototype/sample_texts/f.txt @@ -0,0 +1 @@ +Final Fantasy VII \ No newline at end of file diff --git a/tasks/backend_prototype/sample_texts/g.txt b/tasks/backend_prototype/sample_texts/g.txt new file mode 100644 index 0000000..206a174 --- /dev/null +++ b/tasks/backend_prototype/sample_texts/g.txt @@ -0,0 +1 @@ +God Of War \ No newline at end of file diff --git a/tasks/backend_prototype/sample_texts/h.txt b/tasks/backend_prototype/sample_texts/h.txt new file mode 100644 index 0000000..c534e9e --- /dev/null +++ b/tasks/backend_prototype/sample_texts/h.txt @@ -0,0 +1 @@ +Hollow Knight \ No newline at end of file diff --git a/tasks/backend_prototype/sample_texts/i.txt b/tasks/backend_prototype/sample_texts/i.txt new file mode 100644 index 0000000..b71aca3 --- /dev/null +++ b/tasks/backend_prototype/sample_texts/i.txt @@ -0,0 +1 @@ +It takes two \ No newline at end of file diff --git a/tasks/backend_prototype/sample_texts/j.txt b/tasks/backend_prototype/sample_texts/j.txt new file mode 100644 index 0000000..4e87bac --- /dev/null +++ b/tasks/backend_prototype/sample_texts/j.txt @@ -0,0 +1 @@ +Just Cause 2 \ No newline at end of file diff --git a/tasks/backend_prototype/worker.go b/tasks/backend_prototype/worker.go new file mode 100644 index 0000000..21e17ec --- /dev/null +++ b/tasks/backend_prototype/worker.go @@ -0,0 +1,23 @@ +package main + +import ( + "context" + "log" + "path/filepath" + "time" +) + +type FileJob struct { + Path string +} + +func startWorker(id int, jobs <-chan FileJob, d1tvs [][]float64, timeout time.Duration) { //worker function inside a goroutine, with paramters that are id, recieve channel of jobs, multi-dimensional slice, max time for worker to process a file + for job := range jobs { //until all jobs are done + ctx, cancel := context.WithTimeout(context.Background(), timeout) //context that starts from context.Background() and cancels after timeout duration + err := processFile(ctx, id, job.Path, d1tvs) + cancel() //don't use defer cancel since we don't want to lead to delayed cleanup + if err != nil { + log.Printf("[Worker-%d] Error processing %s: %v", id, filepath.Base(job.Path), err) + } + } +} diff --git a/tasks/crud/README.md b/tasks/crud/README.md new file mode 100644 index 0000000..48efe33 --- /dev/null +++ b/tasks/crud/README.md @@ -0,0 +1,77 @@ +# FileNest CRUD API + +This is a simple RESTful API for managing file metadata, built in Go Language along with MongoDB. It provides endpoints to create, read, update, and delete file metadata records. + +## Features + +- Add new file metadata (Create) +- Retrieve all files or a file by ID (Read) +- Update file metadata by ID (Update) +- Delete file metadata by ID (Delete) +- MongoDB Atlas integration + +## Folder Structure + +``` +tasks/crud/ +├── controllers.go # HTTP handlers for CRUD operations +├── db.go # MongoDB connection and initialization +├── main.go # Entry point, router setup +├── models.go # File metadata model definition +├── go.mod # Go module definition +├── go.sum # Go dependencies +└── README.md # This file +``` + +## Prerequisites + +- Go 1.21 or higher +- MongoDB Atlas cluster (or local MongoDB instance) +- Set your MongoDB URI in `db.go` if needed + +## Getting Started + +1. **Clone the repository:** + ```sh + git clone https://github.com/Abhinav-Kumar2/FileNest.git + cd FileNest/tasks/crud + ``` + +2. **Install dependencies:** + ```sh + go get go.mongodb.org/mongo-driver@latest + go mod tidy + ``` + +3. **Run the server:** + ```sh + go run main.go controllers.go db.go models.go + ``` + + The API will be available at `http://localhost:8000/api/files`. + +## API Endpoints + +| Method | Endpoint | Description | +|--------|----------------------|----------------------------| +| POST | `/api/files` | Create a new file metadata | +| GET | `/api/files` | Get all files | +| GET | `/api/files/{id}` | Get file by ID | +| PUT | `/api/files/{id}` | Update file by ID | +| DELETE | `/api/files/{id}` | Delete file by ID | + +### Example File Metadata JSON + +```json +{ + "filename": "example.pdf", + "filepath": "/files/example.pdf", + "filesize": 123456, + "content_type": "application/pdf" +} +``` + +## Notes + +- The MongoDB connection string is hardcoded in [`db.go`](db.go). Update it to match your environment. +- The API uses the `fileNest` database and `files` collection by default. \ No newline at end of file diff --git a/tasks/crud/controllers.go b/tasks/crud/controllers.go new file mode 100644 index 0000000..0129643 --- /dev/null +++ b/tasks/crud/controllers.go @@ -0,0 +1,97 @@ +package main + +import ( + "context" + "encoding/json" //for encoding/decoding json requests and responses + "fmt" + "net/http" + "time" + + "github.com/gorilla/mux" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func createFile(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var file FileMetadata + err := json.NewDecoder(r.Body).Decode(&file) + if err != nil { + fmt.Println("Error decoding request body:", err) + http.Error(w, "Invalid request body", http.StatusBadRequest) + return + } + file.CreatedAt = time.Now() + file.UpdatedAt = time.Now() + insertResult, err := fileCollection.InsertOne(context.TODO(), file) + if err != nil { + fmt.Println("Error inserting into database:", err) + http.Error(w, "Failed to insert data", http.StatusInternalServerError) + return + } + json.NewEncoder(w).Encode(insertResult.InsertedID) +} + +func getAllFiles(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + cursor, err := fileCollection.Find(context.TODO(), bson.D{}) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer cursor.Close(context.TODO()) //ensure the cursor is closed after use + + var files []FileMetadata + + for cursor.Next(context.TODO()) { + var file FileMetadata + _ = cursor.Decode(&file) + files = append(files, file) + } + json.NewEncoder(w).Encode(files) +} + +func getFileByID(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + params := mux.Vars(r) + id, _ := primitive.ObjectIDFromHex(params["id"]) + + var file FileMetadata + err := fileCollection.FindOne(context.TODO(), bson.M{"_id": id}).Decode(&file) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + json.NewEncoder(w).Encode(file) +} + +func updateFile(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + params := mux.Vars(r) + id, _ := primitive.ObjectIDFromHex(params["id"]) + + var updated FileMetadata + _ = json.NewDecoder(r.Body).Decode(&updated) + updated.UpdatedAt = time.Now() + + update := bson.M{"$set": updated} + result, err := fileCollection.UpdateOne(context.TODO(), bson.M{"_id": id}, update) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + json.NewEncoder(w).Encode(result) +} + +func deleteFile(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + params := mux.Vars(r) + id, _ := primitive.ObjectIDFromHex(params["id"]) + + result, err := fileCollection.DeleteOne(context.TODO(), bson.M{"_id": id}) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + json.NewEncoder(w).Encode(result) +} diff --git a/tasks/crud/db.go b/tasks/crud/db.go new file mode 100644 index 0000000..1abdbad --- /dev/null +++ b/tasks/crud/db.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" //something like control signals that handle how much time a request should take or so + "fmt" //i/o in go + "log" + + "go.mongodb.org/mongo-driver/mongo" //official mongo driver for go + "go.mongodb.org/mongo-driver/mongo/options" //helps in configuring the connection to mongodb +) + +func db() *mongo.Client { //creates a new mongo client and returns it + clientOptions := options.Client().ApplyURI("mongodb+srv://posttoabhinavk:iy4AdEP9Bv5CVhwa@cluster0.t0oc6mk.mongodb.net/") //you can change the URI according to your mongodb cluster + + client, err := mongo.Connect(context.TODO(), clientOptions) //connects my Go program to MongoDB instance, client here will hold the client object while error will check while trying to connect + //context.TODO is like non-nil empty context when not sure what context to use + if err != nil { //if there is any sort of error, connection closes and error is shown + log.Fatal(err) + } + + err = client.Ping(context.TODO(), nil) //just checking the connection to MongoDB is working or not, nil cuz we are not making any changes to URI or someting + if err != nil { + log.Fatal(err) + } + + fmt.Println("Connected to MongoDB") //printing message to confirm connection is done + return client +} + +var fileCollection *mongo.Collection //global variable to hold all files metadata in MongoDB + +func init() { //similar to C, runs before main + client := db() //calling the above db function + fileCollection = client.Database("fileNest").Collection("files") //allows you to call files in filenest database using filecollecion var +} diff --git a/tasks/crud/go.mod b/tasks/crud/go.mod new file mode 100644 index 0000000..2e61ae5 --- /dev/null +++ b/tasks/crud/go.mod @@ -0,0 +1,21 @@ +module filenest + +go 1.24.4 + +require ( + github.com/gorilla/mux v1.8.1 + go.mongodb.org/mongo-driver v1.17.4 +) + +require ( + github.com/golang/snappy v0.0.4 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/text v0.17.0 // indirect +) diff --git a/tasks/crud/go.sum b/tasks/crud/go.sum new file mode 100644 index 0000000..027c158 --- /dev/null +++ b/tasks/crud/go.sum @@ -0,0 +1,52 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw= +go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/tasks/crud/main.go b/tasks/crud/main.go new file mode 100644 index 0000000..b1c0ebf --- /dev/null +++ b/tasks/crud/main.go @@ -0,0 +1,24 @@ +package main //sort of an enry point which helps in turning the entire file into an executable binary + +import ( + "log" //for logging errors/server messages + "net/http" //built in server library for Go, to handle requests and start server + + "github.com/gorilla/mux" //popular routing go library, useful for syntax like /files/{id} +) + +func main() { + router := mux.NewRouter() //new router instance to control all requests + + route := router.PathPrefix("/api").Subrouter() //tells us that this router is handling all api requests, instead of justs starting with /files + + //All Necessary REST API endpoints + + route.HandleFunc("/files", createFile).Methods("POST") + route.HandleFunc("/files", getAllFiles).Methods("GET") + route.HandleFunc("/files/{id}", getFileByID).Methods("GET") + route.HandleFunc("/files/{id}", updateFile).Methods("PUT") + route.HandleFunc("/files/{id}", deleteFile).Methods("DELETE") + + log.Fatal(http.ListenAndServe(":8000", router)) //strts server at port 8000 and prints any errors that doesnt allow serevr to start +} diff --git a/tasks/crud/models.go b/tasks/crud/models.go new file mode 100644 index 0000000..5aad67d --- /dev/null +++ b/tasks/crud/models.go @@ -0,0 +1,17 @@ +package main + +import ( + "time" //helps in created at and updated at stuff + + "go.mongodb.org/mongo-driver/bson/primitive" //mongodb uses ID as primitive object ID +) + +type FileMetadata struct { //defining the file metdaata structure + ID primitive.ObjectID `json:"id,omitempty" bson:"_id,omitempty"` + FileName string `json:"filename" db:"filename"` + FilePath string `json:"filepath" db:"filepath"` + FileSize int64 `json:"filesize" db:"filesize"` + ContentType string `json:"content_type" db:"content_type"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} diff --git a/tasks/messaging_client/go.mod b/tasks/messaging_client/go.mod new file mode 100644 index 0000000..b3eeb6f --- /dev/null +++ b/tasks/messaging_client/go.mod @@ -0,0 +1,3 @@ +module messaging_client + +go 1.24.4 diff --git a/tasks/messaging_client/main.go b/tasks/messaging_client/main.go new file mode 100644 index 0000000..6b4b13c --- /dev/null +++ b/tasks/messaging_client/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "bufio" //buffered I/O to read from terminal + "flag" // command-line flag parsing + "fmt" + "net" //networking for UDP(User Datagram Protocol) sockets + "os" + "os/signal" //to get os singals like ctrl + c for this one + "strings" + "syscall" +) + +func main() { + localPort := flag.Int("local-port", 8000, "Local UDP port to bind") //port to bind to locally + targetPort := flag.Int("target-port", 8001, "Target UDP port to send messages to") //target port to send the messages to + targetIP := flag.String("target-ip", "127.0.0.1", "Target IP address to send messages to") //target IP address to send the messages to + flag.Parse() + + targetAddr := net.JoinHostPort(*targetIP, fmt.Sprintf("%d", *targetPort)) //joining the target Ip and port in the form of host:port + + localAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf(":%d", *localPort)) //Parsing the text form of an address and turning it into a structured data object we can give to the kernel + if err != nil { + fmt.Println("Failed to resolve local address:", err) + return + } + + remoteAddr, err := net.ResolveUDPAddr("udp", targetAddr) + if err != nil { + fmt.Println("Failed to resolve remote address:", err) + return + } + + conn, err := net.ListenUDP("udp", localAddr) //binding the UDP socket to the local address + if err != nil { + fmt.Println("Failed to bind UDP socket:", err) + return + } + defer conn.Close() //ensuring the socket closes when the main exits + + fmt.Printf("Binding to %s\n", localAddr.String()) + fmt.Printf("Target: %s\n", remoteAddr.String()) + fmt.Println(">> Type your message and press Enter (Ctrl+C to quit)") + + sigChan := make(chan os.Signal, 1) //channel to handle ctrl + c + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) //here, os.Interrupt is for Ctrl+C and syscall.SIGTERM is for termination signals, and signal.notify tells go to send these to sigchan channnel + + go func() { //concurrent goroutine to wait for udp datagram input, also defined and called the function at once + buf := make([]byte, 1024) //buffer holding an array of 1024 bytes to read incoming messages + for { + n, addr, err := conn.ReadFromUDP(buf) //putting the message in buf, n is bytes here + if err != nil { + fmt.Println("Error receiving:", err) + continue + } + fmt.Printf("\n>> Received from %s: %s\n", addr.String(), strings.TrimSpace(string(buf[:n]))) //buf[:n] slices the part to show what has been received, string converts bytes into go string, and trimspace takes care of new lines and extra space + fmt.Print(">> ") + } + }() + + scanner := bufio.NewScanner(os.Stdin) //scanner to read input from teh terminal + + go func() { //goroutine for sending messages, concurrency is used so as we don't block the rest of the program(from getting the messages) + for { + fmt.Print(">> ") + if !scanner.Scan() { //waiting till enter + break + } + text := strings.TrimSpace(scanner.Text()) //helps in removing extra whitespaces + if text == "" { //to handle if we press enter without typing anything + fmt.Println("Empty message, please type something.") + continue + } + _, err := conn.WriteToUDP([]byte(text), remoteAddr) //converting my string to a slice of bytes and sending to remoteaddr + if err != nil { + fmt.Println("Error sending:", err) + } + } + }() + + <-sigChan //waiting for ctrl + c/termination signal + fmt.Println("\nExiting...") +} diff --git a/tasks/mnist_classifier/README.md b/tasks/mnist_classifier/README.md new file mode 100644 index 0000000..5345d15 --- /dev/null +++ b/tasks/mnist_classifier/README.md @@ -0,0 +1,60 @@ +# MNIST Classifier + +This project contains a machine learning pipeline for training and evaluating a classifier on the MNIST dataset. It is organized for reproducible experiments and supports configuration management and experiment tracking. + +## Project Structure + +``` +mnist_classifier/ +├── outputs/ +│ └── /