Skip to content

Commit

Permalink
update after trying to run locally
Browse files Browse the repository at this point in the history
  • Loading branch information
kenoir committed Jan 6, 2025
1 parent f1f6c14 commit d773ec5
Show file tree
Hide file tree
Showing 6 changed files with 63 additions and 6 deletions.
16 changes: 16 additions & 0 deletions pipeline/relation_embedder/batcher/local.docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Run locally using `docker compose build lambda && docker compose run --rm --service-ports lambda`
services:
lambda:
build:
context: .
dockerfile: Dockerfile
target: lambda_rie
volumes:
- ~/.aws:/root/.aws
ports:
- "9000:8080"
environment:
- AWS_PROFILE=platform-developer
- metrics_namespace=catalogue-dev_batcher
- use_downstream=stdio
- max_batch_size=100
40 changes: 40 additions & 0 deletions pipeline/relation_embedder/batcher/scripts/run_local.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/usr/bin/env bash

set -euo pipefail

if [ "$#" -lt 1 ]; then
echo "Usage: $0 <PIPELINE_DATE> [--skip-build]"
exit 1
fi

export PIPELINE_DATE=$1
SKIP_BUILD=false
if [ "$#" -eq 2 ] && [ "$2" == "--skip-build" ]; then
SKIP_BUILD=true
fi

PROJECT_NAME="batcher"
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"/..

# Read template.env, substitute variables, and write to .env
if [ -f template.env ]; then
envsubst < template.env > .env
fi

# Build the project, skipping if requested
if [ "$SKIP_BUILD" = true ]; then
echo "Skipping build"
else
pushd ../../..
sbt "project $PROJECT_NAME" ";stage"
popd
fi

# Build the docker image
docker compose -f local.docker-compose.yml \
build lambda

# Run the docker image
docker compose -f local.docker-compose.yml \
run --rm --service-ports lambda
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,5 @@ object LambdaMain
)

override def processT(t: List[String]): Future[String] =
pathsProcessor(t).map(_ => "Done")
pathsProcessor(t.map(PathFromString)).map(_ => "Done")
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package weco.pipeline.batcher
import grizzled.slf4j.Logging
import org.apache.pekko.stream.Materializer
import weco.lambda.Downstream
import weco.json.JsonUtil._

Expand Down Expand Up @@ -127,6 +126,6 @@ object PathsProcessor {
def apply(
downstream: Downstream,
maxBatchSize: Int
)(implicit ec: ExecutionContext, mat: Materializer): PathsProcessor =
)(implicit ec: ExecutionContext): PathsProcessor =
new PathsProcessor(downstream, maxBatchSize)
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ trait BatcherConfigurable extends LambdaConfigurable[BatcherConfig] {
BatcherConfig(
maxBatchSize = rawConfig.requireInt("batcher.max_batch_size"),
downstreamTarget = {
rawConfig.requireString("relation_embedder.use_downstream") match {
rawConfig.requireString("batcher.use_downstream") match {
case "sns" => SNS(buildSNSConfig(rawConfig))
case "stdio" => StdOut
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ PROJECT_NAME="relation_embedder"
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"/..

# Read .template.env, substitute variables, and write to .env
envsubst < template.env > .env
# Read template.env, substitute variables, and write to .env
if [ -f template.env ]; then
envsubst < template.env > .env
fi

# Build the project, skipping if requested
if [ "$SKIP_BUILD" = true ]; then
Expand Down

0 comments on commit d773ec5

Please sign in to comment.