Skip to content

Commit

Permalink
made sure that the student clubs are localised (#438)
Browse files Browse the repository at this point in the history
* made sure that the student clubs are localised

* fixed most of the bugs in the sv scraper and related issues

* fixed a remakingly stupid typo

* made sure that the manual migration is up to date

* tested a different filtering which does not trigger testcases

* made sure that the testcase is more happy

* fixed formatting issues
  • Loading branch information
CommanderStorm authored Sep 24, 2024
1 parent 86288f4 commit ede8b2e
Show file tree
Hide file tree
Showing 17 changed files with 672 additions and 423 deletions.
2 changes: 1 addition & 1 deletion server/api/generate.bash
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ pushd $BASEDIR > /dev/null

echo updating the generated files
export PATH="$PATH:$(go env GOPATH)/bin"
buf mod update || exit 1
buf dep update || exit 1
buf generate || exit 1

echo making sure the openapi document points to the valid api
Expand Down
23 changes: 12 additions & 11 deletions server/api/installBuf.bash
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,21 @@ BASEDIR=$(dirname "$0")
echo "making sure that this script is run from $BASEDIR"
pushd "$BASEDIR" > /dev/null || exit

go work use ./

echo downloading...
go get github.com/bufbuild/buf/cmd/buf \
github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway \
github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 \
google.golang.org/protobuf/cmd/protoc-gen-go \
google.golang.org/grpc/cmd/protoc-gen-go-grpc
go get github.com/bufbuild/buf/cmd/buf
go get github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway
go get github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2
go get google.golang.org/protobuf/cmd/protoc-gen-go
go get google.golang.org/grpc/cmd/protoc-gen-go-grpc

echo installing...
go install \
github.com/bufbuild/buf/cmd/buf \
github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway \
github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 \
google.golang.org/protobuf/cmd/protoc-gen-go \
google.golang.org/grpc/cmd/protoc-gen-go-grpc
GO111MODULE=on go install github.com/bufbuild/buf/cmd/buf
go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway
go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2
go install google.golang.org/protobuf/cmd/protoc-gen-go
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc


echo tidiing up
Expand Down
772 changes: 426 additions & 346 deletions server/api/tumdev/campus_backend.pb.go

Large diffs are not rendered by default.

18 changes: 18 additions & 0 deletions server/api/tumdev/campus_backend.pb.gw.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 14 additions & 1 deletion server/api/tumdev/campus_backend.proto
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,17 @@ message GetCanteenHeadCountReply {
// A time stamp indicating how up to date the response is. Only valid in case percent != -1.
google.protobuf.Timestamp timestamp = 4;
}
message ListStudentClubRequest {}

enum Language {
German = 0;
English = 1;
}

message ListStudentClubRequest {
// Language of the student clubs and categories
// Defaults to german
optional Language language = 1;
}
message ListStudentClubReply {
repeated StudentClubCollection collections = 1;
}
Expand All @@ -582,4 +592,7 @@ message StudentClubCollection {
string title = 1;
string description = 2;
repeated StudentClub clubs = 3;
// id of the collection.
// Might not be stable over time because of scraping
uint64 unstable_collection_id = 4;
}
27 changes: 27 additions & 0 deletions server/api/tumdev/campus_backend.swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -805,6 +805,20 @@
}
}
},
"parameters": [
{
"name": "language",
"description": "Language of the student clubs and categories\nDefaults to german",
"in": "query",
"required": false,
"type": "string",
"enum": [
"German",
"English"
],
"default": "German"
}
],
"tags": [
"Campus"
]
Expand Down Expand Up @@ -1172,6 +1186,14 @@
}
}
},
"apiLanguage": {
"type": "string",
"enum": [
"German",
"English"
],
"default": "German"
},
"apiListAvailableCanteenTagsReply": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -1622,6 +1644,11 @@
"type": "object",
"$ref": "#/definitions/apiStudentClub"
}
},
"unstableCollectionId": {
"type": "string",
"format": "uint64",
"title": "id of the collection.\nMight not be stable over time because of scraping"
}
}
},
Expand Down
5 changes: 4 additions & 1 deletion server/backend/cron/cronjobs.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package cron
import (
"time"

pb "github.com/TUM-Dev/Campus-Backend/server/api/tumdev"

"github.com/TUM-Dev/Campus-Backend/server/model"
"github.com/mmcdole/gofeed"
log "github.com/sirupsen/logrus"
Expand Down Expand Up @@ -69,7 +71,8 @@ func (c *CronService) Run() error {
// Run each job in a separate goroutine, so we can parallelize them
switch cronjob.Type.String {
case StudentClubType:
g.Go(func() error { return c.studentClubCron() })
g.Go(func() error { return c.studentClubCron(pb.Language_German) })
g.Go(func() error { return c.studentClubCron(pb.Language_English) })
case NewsType:
// if this is not copied here, this may not be threads save due to go's guarantees
// loop variable cronjob captured by func literal (govet)
Expand Down
25 changes: 19 additions & 6 deletions server/backend/cron/student_clubs.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"io"
"strings"

pb "github.com/TUM-Dev/Campus-Backend/server/api/tumdev"

"github.com/TUM-Dev/Campus-Backend/server/backend/cron/student_club_parsers"
"gorm.io/gorm"

Expand All @@ -20,8 +22,8 @@ const (
StudentClubImageDirectory = "student_club/"
)

func (c *CronService) studentClubCron() error {
body, err := student_club_parsers.DownloadHtml("https://www.sv.tum.de/sv/hochschulgruppen/")
func (c *CronService) studentClubCron(language pb.Language) error {
body, err := student_club_parsers.DownloadHtml(svUrl(language))
defer func(Body io.ReadCloser) {
if err := Body.Close(); err != nil {
log.WithError(err).Error("Error while closing body")
Expand All @@ -37,27 +39,31 @@ func (c *CronService) studentClubCron() error {

// save the result of the previous steps (🎉)
if err := c.db.Transaction(func(tx *gorm.DB) error {
if err := tx.Where("1 = 1").Delete(&model.StudentClub{}).Error; err != nil {
if err := tx.Where("language = ?", language.String()).Delete(&model.StudentClub{}).Error; err != nil {
return err
}
if err := tx.Where("1 = 1").Delete(&model.StudentClubCollection{}).Error; err != nil {
if err := tx.Where("language = ?", language.String()).Delete(&model.StudentClubCollection{}).Error; err != nil {
return err
}
nameToCollectionID := make(map[string]uint)
for _, scrapedCollection := range scrapedCollections {
collection := model.StudentClubCollection{
ID: scrapedCollection.Name,
Name: scrapedCollection.Name,
Language: language.String(),
Description: scrapedCollection.Description,
}
if err := tx.Create(&collection).Error; err != nil {
return err
}
nameToCollectionID[collection.Name] = collection.ID
}
for _, scrapedClub := range scrapedClubs {
club := model.StudentClub{
Language: language.String(),
Name: scrapedClub.Name,
Description: scrapedClub.Description,
LinkUrl: scrapedClub.LinkUrl,
StudentClubCollectionID: scrapedClub.Collection,
StudentClubCollectionID: nameToCollectionID[scrapedClub.Collection],
}
if scrapedClub.ImageUrl.Valid {
file, err := saveImageTo(tx, scrapedClub.ImageUrl.String, StudentClubImageDirectory)
Expand All @@ -79,6 +85,13 @@ func (c *CronService) studentClubCron() error {
return nil
}

func svUrl(language pb.Language) string {
if language == pb.Language_English {
return "https://www.sv.tum.de/en/sv/student-groups/"
}
return "https://www.sv.tum.de/sv/hochschulgruppen/"
}

// saveImage saves an image to the database, so it can be downloaded by another cronjob and returns the file
func saveImageTo(tx *gorm.DB, url string, path string) (*model.File, error) {
seps := strings.SplitAfter(url, ".")
Expand Down
12 changes: 9 additions & 3 deletions server/backend/cron/student_clubs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"testing"

pb "github.com/TUM-Dev/Campus-Backend/server/api/tumdev"

"github.com/TUM-Dev/Campus-Backend/server/model"
"github.com/TUM-Dev/Campus-Backend/server/utils"
"github.com/stretchr/testify/require"
Expand All @@ -15,9 +17,13 @@ func TestCronService_studentClubCron(t *testing.T) {
db := utils.SetupTestContainer(ctx, t)
service := New(db)
var clubs []model.StudentClub
db.WithContext(ctx).Find(&clubs)
require.NoError(t, db.WithContext(ctx).Find(&clubs).Error)
require.Equal(t, []model.StudentClub{}, clubs)
require.NoError(t, service.studentClubCron())
db.WithContext(ctx).Find(&clubs)
require.NoError(t, service.studentClubCron(pb.Language_German))
require.NoError(t, db.WithContext(ctx).Find(&clubs).Error)
require.True(t, len(clubs) > 0)
require.NoError(t, db.WithContext(ctx).Where("1=1").Delete(&model.StudentClub{}).Error)
require.NoError(t, service.studentClubCron(pb.Language_English))
require.NoError(t, db.WithContext(ctx).Find(&clubs).Error)
require.True(t, len(clubs) > 0)
}
76 changes: 76 additions & 0 deletions server/backend/migration/20241023000000.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package migration

import (
"github.com/go-gormigrate/gormigrate/v2"
"github.com/guregu/null"
"gorm.io/gorm"
)

// StudentClub stores a student Club
type newStudentClub struct {
gorm.Model
Name string
Language string `gorm:"type:enum('German','English');default:'German'"`
Description null.String
LinkUrl null.String `gorm:"type:varchar(190);unique;"`
ImageID null.Int
Image *File `gorm:"foreignKey:ImageID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"`
ImageCaption null.String
StudentClubCollectionID uint
StudentClubCollection newStudentClubCollection `gorm:"foreignKey:StudentClubCollectionID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"`
}

// TableName sets the insert table name for this struct type
func (n *newStudentClub) TableName() string {
return "student_clubs"
}

type newStudentClubCollection struct {
gorm.Model
Name string `gorm:"type:varchar(100)"`
Language string `gorm:"type:enum('German','English');default:'German'"`
Description string
}

// TableName sets the insert table name for this struct type
func (n *newStudentClubCollection) TableName() string {
return "student_club_collections"
}

// migrate20241023000000
// - made sure that student clubs are localised
func migrate20241023000000() *gormigrate.Migration {
return &gormigrate.Migration{
ID: "20241023000000",
Migrate: func(tx *gorm.DB) error {
if err := tx.Migrator().DropTable(InitialStudentClub{}); err != nil {
return err
}
if err := tx.Migrator().DropTable(InitialStudentClubCollection{}); err != nil {
return err
}
if err := tx.Migrator().AutoMigrate(newStudentClubCollection{}); err != nil {
return err
}
if err := tx.Migrator().AutoMigrate(newStudentClub{}); err != nil {
return err
}
return nil
},
Rollback: func(tx *gorm.DB) error {
if err := tx.Migrator().DropTable(newStudentClub{}); err != nil {
return err
}
if err := tx.Migrator().DropTable(newStudentClubCollection{}); err != nil {
return err
}
if err := tx.Migrator().AutoMigrate(InitialStudentClubCollection{}); err != nil {
return err
}
if err := tx.Migrator().AutoMigrate(InitialStudentClub{}); err != nil {
return err
}
return nil
},
}
}
1 change: 1 addition & 0 deletions server/backend/migration/migration.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ func manualMigrate(db *gorm.DB) error {
migrate20240512000000(),
migrate20240706000000(),
migrate20240824000000(),
migrate20241023000000(),
}
return gormigrate.New(db, gormigrateOptions, migrations).Migrate()
}
Expand Down
Loading

0 comments on commit ede8b2e

Please sign in to comment.