diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000000..8217c2e591 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,45 @@ +{ + "extends": ["eslint:recommended", "plugin:react/recommended", "plugin:@typescript-eslint/recommended"], + "plugins": ["react", "@typescript-eslint"], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaFeatures": { + "jsx": true + } + }, + + "env": { + "browser": true, + "es2021": true, + "node": true + }, + + "rules": { + "react/prop-types": 0, + "@typescript-eslint/no-unused-vars": ["error", {"varsIgnorePattern": "_"}], + "@typescript-eslint/explicit-module-boundary-types": "off", + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-empty-interface": 0, + "@typescript-eslint/ban-types": ["error", {"extendDefaults": true, "types": {"{}": false, "Object": false}}] + }, + "settings": { + "react": { + "version": "16.12" + } + }, + "overrides": [ + { + "files": ["frontend/chat-plugin/**/*.tsx"], + "settings": { + "react": { + "pragma": "h" + }, + "import/resolver": { + "node": { + "extensions": [".ts", ".tsx"] + } + } + } + } + ] +} diff --git a/VERSION b/VERSION index 0d91a54c7d..1d0ba9ea18 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 +0.4.0 diff --git a/WORKSPACE b/WORKSPACE index 063b1ef4f1..97010f9cdf 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -177,10 +177,9 @@ load( container_pull( name = "java_base", - digest = "sha256:9f080b14f9d2c42b7a753169daf5ee7f6c0cbaa36d51ab4390a132941df0b111", - registry = "index.docker.io", - repository = "library/openjdk", - tag = "11.0.3-jre-slim", + registry = "gcr.io", + repository = "distroless/java", + tag = "11", ) container_pull( diff --git a/backend/BUILD b/backend/BUILD index a55d6f2287..a4c5e7acd5 100644 --- a/backend/BUILD +++ b/backend/BUILD @@ -23,30 +23,6 @@ java_library( ], ) -java_library( - name = "channel", - exports = [ - "//backend/avro/communication:channel", - "//lib/java/kafka/schema:application-communication-channels", - ], -) - -java_library( - name = "message", - exports = [ - "//backend/avro/communication:message", - "//lib/java/kafka/schema:application-communication-messages", - ], -) - -java_library( - name = "metadata", - exports = [ - "//backend/avro/communication:metadata-action", - "//lib/java/kafka/schema:application-communication-metadata", - ], -) - java_library( name = "read-receipt", exports = [ diff --git a/backend/api/admin/BUILD b/backend/api/admin/BUILD index 0ab199c815..1b00171ee5 100644 --- a/backend/api/admin/BUILD +++ b/backend/api/admin/BUILD @@ -5,10 +5,9 @@ load("//tools/build:container_push.bzl", "container_push") app_deps = [ "//backend:base_app", "//:springboot_actuator", - "//backend:channel", + "//backend/model/channel:channel", "//backend:tag", "//backend:webhook", - "//lib/java/payload", "//lib/java/uuid", "//lib/java/spring/auth:spring-auth", "//lib/java/spring/web:spring-web", diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/ChannelsController.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/ChannelsController.java index d8944ca9e2..2f8b09bcdf 100644 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/ChannelsController.java +++ b/backend/api/admin/src/main/java/co/airy/core/api/admin/ChannelsController.java @@ -2,16 +2,15 @@ import co.airy.avro.communication.Channel; import co.airy.avro.communication.ChannelConnectionState; -import co.airy.core.api.admin.dto.ChannelMetadata; -import co.airy.core.api.admin.payload.AvailableChannelPayload; -import co.airy.core.api.admin.payload.AvailableChannelsRequestPayload; -import co.airy.core.api.admin.payload.AvailableChannelsResponsePayload; import co.airy.core.api.admin.payload.ChannelsResponsePayload; -import co.airy.core.api.admin.payload.ConnectChannelRequestPayload; -import co.airy.core.api.admin.payload.DisconnectChannelRequestPayload; -import co.airy.payload.response.EmptyResponsePayload; -import co.airy.payload.response.RequestErrorResponsePayload; +import co.airy.kafka.schema.application.ApplicationCommunicationChannels; +import co.airy.model.channel.ChannelPayload; +import co.airy.spring.web.payload.EmptyResponsePayload; import co.airy.uuid.UUIDv5; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; @@ -19,136 +18,62 @@ import org.springframework.web.bind.annotation.RestController; import javax.validation.Valid; -import java.util.HashMap; +import javax.validation.constraints.NotNull; import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.UUID; +import static co.airy.model.channel.ChannelPayload.fromChannel; import static java.util.stream.Collectors.toList; @RestController public class ChannelsController { - private final Stores stores; + private final KafkaProducer producer; + private final String applicationCommunicationChannels = new ApplicationCommunicationChannels().name(); - private final Map sourceMap = new HashMap<>(); - - public ChannelsController(Stores stores, List sources) { - for (Source source : sources) { - sourceMap.put(source.getIdentifier(), source); - } + public ChannelsController(Stores stores, KafkaProducer producer) { this.stores = stores; + this.producer = producer; } @PostMapping("/channels.list") - ResponseEntity connectedChannels() { - final Map channelsMap = stores.getChannelsMap(); - - return ResponseEntity.ok( - new ChannelsResponsePayload( - channelsMap.values() - .stream() - .filter((channel -> ChannelConnectionState.CONNECTED.equals(channel.getConnectionState()))) - .map(Mapper::fromChannel) - .collect(toList()) - ) - ); - } - - @PostMapping("/channels.explore") - ResponseEntity listChannels(@RequestBody @Valid AvailableChannelsRequestPayload requestPayload) { - final String sourceIdentifier = requestPayload.getSource(); - - final Source source = sourceMap.get(sourceIdentifier); - - if (source == null) { - return ResponseEntity.badRequest().body(new RequestErrorResponsePayload(String.format("source %s not implemented", sourceIdentifier))); - } - - final List availableChannels; + ResponseEntity listChannels() { + final List channels = stores.getChannels(); - try { - availableChannels = source.getAvailableChannels(requestPayload.getToken()); - } catch (SourceApiException e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new RequestErrorResponsePayload(e.getMessage())); - } - - final Map channelsMap = stores.getChannelsMap(); - final List connectedSourceIds = channelsMap.values() - .stream() - .filter((channel -> ChannelConnectionState.CONNECTED.equals(channel.getConnectionState()))) - .map(Channel::getSourceChannelId) - .collect(toList()); - - return ResponseEntity.ok( - new AvailableChannelsResponsePayload( - availableChannels.stream() - .map((channel) -> AvailableChannelPayload.builder() - .sourceChannelId(channel.getSourceChannelId()) - .name(channel.getName()) - .imageUrl(channel.getImageUrl()) - .connected(connectedSourceIds.contains(channel.getSourceChannelId())) - .build() - ) - .collect(toList()) - ) - ); + return ResponseEntity.ok(new ChannelsResponsePayload(channels.stream() + .map(ChannelPayload::fromChannel) + .collect(toList()))); } - @PostMapping("/channels.connect") - ResponseEntity connectChannel(@RequestBody @Valid ConnectChannelRequestPayload requestPayload) { - final String token = requestPayload.getToken(); - final String sourceChannelId = requestPayload.getSourceChannelId(); - final String sourceIdentifier = requestPayload.getSource(); + @PostMapping("/chatplugin.connect") + ResponseEntity connect(@RequestBody @Valid ConnectChannelRequestPayload requestPayload) { + final String sourceChannelId = requestPayload.getName(); + final String sourceIdentifier = "chat_plugin"; final String channelId = UUIDv5.fromNamespaceAndName(sourceIdentifier, sourceChannelId).toString(); - final Source source = sourceMap.get(sourceIdentifier); - - if (source == null) { - return ResponseEntity.badRequest().body(new RequestErrorResponsePayload(String.format("source %s not implemented", source))); - } - - final Map channelsMap = stores.getChannelsMap(); - final Channel existingChannel = channelsMap.get(channelId); - - if (existingChannel != null && ChannelConnectionState.CONNECTED.equals(existingChannel.getConnectionState())) { - return ResponseEntity.ok(Mapper.fromChannel(existingChannel)); - } - - final ChannelMetadata channelMetadata; - - try { - channelMetadata = source.connectChannel(token, sourceChannelId); - } catch (SourceApiException e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new RequestErrorResponsePayload(e.getMessage())); - } - final Channel channel = Channel.newBuilder() .setId(channelId) .setConnectionState(ChannelConnectionState.CONNECTED) - .setImageUrl(Optional.ofNullable(requestPayload.getImageUrl()).orElse(channelMetadata.getImageUrl())) - .setName(Optional.ofNullable(requestPayload.getName()).orElse(channelMetadata.getName())) .setSource(sourceIdentifier) .setSourceChannelId(sourceChannelId) - .setToken(token) + .setName(requestPayload.getName()) .build(); try { - stores.storeChannel(channel); + producer.send(new ProducerRecord<>(applicationCommunicationChannels, channel.getId(), channel)).get(); } catch (Exception e) { return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build(); } - return ResponseEntity.ok(Mapper.fromChannel(channel)); + return ResponseEntity.ok(fromChannel(channel)); } - @PostMapping("/channels.disconnect") - ResponseEntity disconnectChannel(@RequestBody @Valid DisconnectChannelRequestPayload requestPayload) { + @PostMapping("/chatplugin.disconnect") + ResponseEntity disconnect(@RequestBody @Valid ChannelDisconnectRequestPayload requestPayload) { final String channelId = requestPayload.getChannelId().toString(); - final Map channelsMap = stores.getChannelsMap(); - final Channel channel = channelsMap.get(channelId); + final Channel channel = stores.getConnectedChannelsStore().get(channelId); if (channel == null) { return ResponseEntity.notFound().build(); @@ -158,28 +83,30 @@ ResponseEntity disconnectChannel(@RequestBody @Valid DisconnectChannelRequest return ResponseEntity.accepted().build(); } - final Source source = sourceMap.get(channel.getSource()); - - if (source == null) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body(new RequestErrorResponsePayload(String.format("source %s not implemented", channel.getSource()))); - } - - try { - source.disconnectChannel(channel.getToken(), channel.getSourceChannelId()); - } catch (SourceApiException e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new RequestErrorResponsePayload(e.getMessage())); - } - channel.setConnectionState(ChannelConnectionState.DISCONNECTED); channel.setToken(null); try { - stores.storeChannel(channel); + producer.send(new ProducerRecord<>(applicationCommunicationChannels, channel.getId(), channel)).get(); } catch (Exception e) { return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build(); } return ResponseEntity.ok(new EmptyResponsePayload()); } + +} + +@Data +@NoArgsConstructor +class ConnectChannelRequestPayload { + @NotNull + private String name; +} + +@Data +@NoArgsConstructor +class ChannelDisconnectRequestPayload { + @NotNull + private UUID channelId; } diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/Mapper.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/Mapper.java deleted file mode 100644 index aab37b4cab..0000000000 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/Mapper.java +++ /dev/null @@ -1,16 +0,0 @@ -package co.airy.core.api.admin; - -import co.airy.avro.communication.Channel; -import co.airy.payload.response.ChannelPayload; - -public class Mapper { - public static ChannelPayload fromChannel(Channel channel) { - return ChannelPayload.builder() - .name(channel.getName()) - .id(channel.getId()) - .imageUrl(channel.getImageUrl()) - .source(channel.getSource()) - .sourceChannelId(channel.getSourceChannelId()) - .build(); - } -} diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/Source.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/Source.java deleted file mode 100644 index 168ef1bde8..0000000000 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/Source.java +++ /dev/null @@ -1,15 +0,0 @@ -package co.airy.core.api.admin; - -import co.airy.core.api.admin.dto.ChannelMetadata; - -import java.util.List; - -public interface Source { - String getIdentifier(); - - List getAvailableChannels(String token) throws SourceApiException; - - ChannelMetadata connectChannel(String token, String sourceChannelId) throws SourceApiException; - - void disconnectChannel(String token, String sourceChannelId) throws SourceApiException; -} diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/SourceApiException.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/SourceApiException.java deleted file mode 100644 index 43d0dcb0d1..0000000000 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/SourceApiException.java +++ /dev/null @@ -1,7 +0,0 @@ -package co.airy.core.api.admin; - -public class SourceApiException extends Exception { - public SourceApiException(String errorMessage) { - super(errorMessage); - } -} diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/Stores.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/Stores.java index 45acfd006e..f480fad2a7 100644 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/Stores.java +++ b/backend/api/admin/src/main/java/co/airy/core/api/admin/Stores.java @@ -1,6 +1,7 @@ package co.airy.core.api.admin; import co.airy.avro.communication.Channel; +import co.airy.avro.communication.ChannelConnectionState; import co.airy.avro.communication.Tag; import co.airy.avro.communication.Webhook; import co.airy.kafka.schema.application.ApplicationCommunicationChannels; @@ -12,18 +13,17 @@ import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.state.KeyValueIterator; import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; import org.springframework.beans.factory.DisposableBean; import org.springframework.boot.actuate.health.Health; import org.springframework.boot.actuate.health.HealthIndicator; -import org.springframework.boot.actuate.health.Status; import org.springframework.boot.context.event.ApplicationStartedEvent; import org.springframework.context.ApplicationListener; import org.springframework.stereotype.Component; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.ExecutionException; @Component @@ -33,10 +33,9 @@ public class Stores implements HealthIndicator, ApplicationListener producer; - private final String channelsStore = "channels-store"; + private final String connectedChannelsStore = "connected-channels-store"; private final String tagsStore = "tags-store"; private final String webhooksStore = "webhook-store"; - private final String allChannelsKey = "ALL"; // Using a UUID as the default key for the webhook will make it easier // to add multiple webhooks if that ever becomes a requirement @@ -51,16 +50,12 @@ public Stores(KafkaStreamsWrapper streams, KafkaProducerstream(applicationCommunicationChannels) - .groupBy((k, v) -> allChannelsKey) - .aggregate(HashMap::new, (allKey, channel, channelsMap) -> { - // An external channel id may only be connected once - channelsMap.put(channel.getId(), channel); - return channelsMap; - }, Materialized.as(channelsStore)); + builder.table(applicationCommunicationChannels) + .filter((k, v) -> v.getConnectionState().equals(ChannelConnectionState.CONNECTED), Materialized.as(connectedChannelsStore)); builder.stream(applicationCommunicationWebhooks) .groupBy((webhookId, webhook) -> allWebhooksKey) @@ -71,10 +66,6 @@ private void startStream() { streams.start(builder.build(), appId); } - public ReadOnlyKeyValueStore> getChannelsStore() { - return streams.acquireLocalStore(channelsStore); - } - public ReadOnlyKeyValueStore getWebhookStore() { return streams.acquireLocalStore(webhooksStore); } @@ -83,10 +74,6 @@ public ReadOnlyKeyValueStore getTagsStore() { return streams.acquireLocalStore(tagsStore); } - public void storeChannel(Channel channel) throws ExecutionException, InterruptedException { - producer.send(new ProducerRecord<>(applicationCommunicationChannels, channel.getId(), channel)).get(); - } - public void storeWebhook(Webhook webhook) throws ExecutionException, InterruptedException { webhook.setId(allWebhooksKey); producer.send(new ProducerRecord<>(applicationCommunicationWebhooks, allWebhooksKey, webhook)).get(); @@ -100,10 +87,19 @@ public void deleteTag(Tag tag) { producer.send(new ProducerRecord<>(applicationCommunicationTags, tag.getId(), null)); } - public Map getChannelsMap() { - final ReadOnlyKeyValueStore> channelsStore = getChannelsStore(); + public ReadOnlyKeyValueStore getConnectedChannelsStore() { + return streams.acquireLocalStore(connectedChannelsStore); + } + + public List getChannels() { + final ReadOnlyKeyValueStore store = getConnectedChannelsStore(); + + final KeyValueIterator iterator = store.all(); - return Optional.ofNullable(channelsStore.get(allChannelsKey)).orElse(Map.of()); + List channels = new ArrayList<>(); + iterator.forEachRemaining(kv -> channels.add(kv.value)); + + return channels; } public Webhook getWebhook() { @@ -119,17 +115,12 @@ public void destroy() { } } - @Override - public void onApplicationEvent(ApplicationStartedEvent event) { - startStream(); - } - @Override public Health health() { - getChannelsStore(); + getConnectedChannelsStore(); getWebhookStore(); getTagsStore(); - return Health.status(Status.UP).build(); + return Health.up().build(); } } diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/TagsController.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/TagsController.java index 42952ba928..bb5444e524 100644 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/TagsController.java +++ b/backend/api/admin/src/main/java/co/airy/core/api/admin/TagsController.java @@ -8,8 +8,8 @@ import co.airy.core.api.admin.payload.ListTagsResponsePayload; import co.airy.core.api.admin.payload.TagResponsePayload; import co.airy.core.api.admin.payload.UpdateTagRequestPayload; -import co.airy.payload.response.EmptyResponsePayload; -import co.airy.payload.response.RequestErrorResponsePayload; +import co.airy.spring.web.payload.EmptyResponsePayload; +import co.airy.spring.web.payload.RequestErrorResponsePayload; import org.apache.kafka.streams.state.KeyValueIterator; import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; import org.springframework.http.HttpStatus; diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/WebhooksController.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/WebhooksController.java index ffa9082cbb..d8d4c5cf31 100644 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/WebhooksController.java +++ b/backend/api/admin/src/main/java/co/airy/core/api/admin/WebhooksController.java @@ -43,14 +43,7 @@ public ResponseEntity subscribe(@RequestBody @Valid WebhookSubscriptionPayloa return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(e.getMessage()); } - final GetWebhookResponse webhookResponse = GetWebhookResponse.builder() - .headers(webhook.getHeaders()) - .apiSecret(apiSecret) - .status(webhook.getStatus().toString()) - .url(webhook.getEndpoint()) - .build(); - - return ResponseEntity.status(HttpStatus.OK).body(webhookResponse); + return ResponseEntity.status(HttpStatus.OK).body(fromWebhook(webhook)); } @PostMapping("/webhooks.unsubscribe") @@ -69,14 +62,7 @@ public ResponseEntity unsubscribe() { return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(e.getMessage()); } - final GetWebhookResponse webhookResponse = GetWebhookResponse.builder() - .headers(webhook.getHeaders()) - .apiSecret(webhook.getApiSecret()) - .status(webhook.getStatus().toString()) - .url(webhook.getEndpoint()) - .build(); - - return ResponseEntity.status(HttpStatus.OK).body(webhookResponse); + return ResponseEntity.status(HttpStatus.OK).body(fromWebhook(webhook)); } @PostMapping("/webhooks.info") @@ -87,13 +73,15 @@ public ResponseEntity webhookInfo() { return ResponseEntity.notFound().build(); } - final GetWebhookResponse webhookResponse = GetWebhookResponse.builder() + return ResponseEntity.status(HttpStatus.OK).body(fromWebhook(webhook)); + } + + private GetWebhookResponse fromWebhook(Webhook webhook) { + return GetWebhookResponse.builder() .headers(webhook.getHeaders()) .apiSecret(webhook.getApiSecret()) .status(webhook.getStatus().toString()) .url(webhook.getEndpoint()) .build(); - - return ResponseEntity.status(HttpStatus.OK).body(webhookResponse); } } diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/dto/ChannelMetadata.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/dto/ChannelMetadata.java deleted file mode 100644 index 7e7ca60baf..0000000000 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/dto/ChannelMetadata.java +++ /dev/null @@ -1,16 +0,0 @@ -package co.airy.core.api.admin.dto; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class ChannelMetadata { - private String sourceChannelId; - private String name; - private String imageUrl; -} diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/ChannelsResponsePayload.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/ChannelsResponsePayload.java index 2b2bc63869..49903e25f4 100644 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/ChannelsResponsePayload.java +++ b/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/ChannelsResponsePayload.java @@ -1,6 +1,6 @@ package co.airy.core.api.admin.payload; -import co.airy.payload.response.ChannelPayload; +import co.airy.model.channel.ChannelPayload; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/ConnectChannelRequestPayload.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/ConnectChannelRequestPayload.java deleted file mode 100644 index c791e48b0c..0000000000 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/ConnectChannelRequestPayload.java +++ /dev/null @@ -1,20 +0,0 @@ -package co.airy.core.api.admin.payload; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import javax.validation.constraints.NotNull; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class ConnectChannelRequestPayload { - @NotNull - String source; - @NotNull - String sourceChannelId; - String token; - String name; - String imageUrl; -} diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/chat_plugin/ChatPluginSource.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/chat_plugin/ChatPluginSource.java deleted file mode 100644 index ef5228d9d3..0000000000 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/chat_plugin/ChatPluginSource.java +++ /dev/null @@ -1,32 +0,0 @@ -package co.airy.core.api.admin.sources.chat_plugin; - -import co.airy.core.api.admin.Source; -import co.airy.core.api.admin.dto.ChannelMetadata; -import org.springframework.stereotype.Service; - -import java.util.List; - -@Service -public class ChatPluginSource implements Source { - @Override - public String getIdentifier() { - return "chat_plugin"; - } - - @Override - public List getAvailableChannels(String token) { - return List.of(); - } - - @Override - public ChannelMetadata connectChannel(String token, String sourceChannelId) { - return ChannelMetadata.builder() - .name("Chat plugin") - .sourceChannelId(sourceChannelId) - .build(); - } - - @Override - public void disconnectChannel(String token, String sourceChannelId) { - } -} diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/facebook/FacebookApi.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/facebook/FacebookApi.java deleted file mode 100644 index f9cea3b692..0000000000 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/facebook/FacebookApi.java +++ /dev/null @@ -1,109 +0,0 @@ -package co.airy.core.api.admin.sources.facebook; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpMethod; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; - -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - - -@Service -public class FacebookApi { - private static final String SUBSCRIBED_FIELDS = "messages,messaging_postbacks,messaging_optins,message_deliveries,message_reads,messaging_payments,messaging_pre_checkouts,messaging_checkout_updates,messaging_account_linking,messaging_referrals,message_echoes,messaging_game_plays,standby,messaging_handovers,messaging_policy_enforcement,message_reactions,inbox_labels"; - private final RestTemplate restTemplate = new RestTemplate(); - private final String baseUrl = "https://graph.facebook.com/v3.2"; - private final String pageFields = "fields=id,name_with_location_descriptor,access_token,picture,is_webhooks_subscribed"; - - private final String fbAppId; - private final String fbApiSecret; - private final ObjectMapper objectMapper; - - FacebookApi( - @Value("${facebook.app-id}") String fbAppId, - @Value("${facebook.app-secret}") String fbApiSecret, - ObjectMapper objectMapper - ) { - this.fbAppId = fbAppId; - this.fbApiSecret = fbApiSecret; - this.objectMapper = objectMapper; - } - - public List getAllPagesForUser(String accessToken) throws Exception { - String pagesUrl = String.format(baseUrl + "/me/accounts?%s&access_token=%s", pageFields, accessToken); - - boolean hasMorePages = true; - List pageList = new ArrayList<>(); - while (hasMorePages) { - FbPages fbPages = apiResponse(pagesUrl, HttpMethod.GET, FbPages.class); - if (fbPages.getPaging() != null && fbPages.getPaging().getNext() != null) { - pagesUrl = URLDecoder.decode(fbPages.getPaging().getNext(), StandardCharsets.UTF_8); - } else { - hasMorePages = false; - } - pageList.addAll(fbPages.getData()); - } - return pageList; - } - - public FbPageWithConnectInfo getPageForUser(final String pageId, final String accessToken) throws Exception { - final String pageUrl = String.format(baseUrl + "/%s?%s&access_token=%s", pageId, pageFields, accessToken); - - return apiResponse(pageUrl, HttpMethod.GET, FbPageWithConnectInfo.class); - } - - public void connectPageToApp(String pageToken) throws Exception { - String apiUrl = String.format(baseUrl + "/me/subscribed_apps?access_token=%s&subscribed_fields=%s", pageToken, SUBSCRIBED_FIELDS); - apiResponse(apiUrl, HttpMethod.POST, Map.class); - } - - public String exchangeToLongLivingUserAccessToken(String userAccessToken) throws Exception { - String apiUrl = String.format(baseUrl + "/oauth/access_token?grant_type=fb_exchange_token&client_id=%s&client_secret=%s&fb_exchange_token=%s", fbAppId, fbApiSecret, userAccessToken); - return apiResponse(apiUrl, HttpMethod.GET, FbLongLivingUserAccessToken.class).getAccessToken(); - } - - - private T apiResponse(String url, HttpMethod method, Class clazz) throws Exception { - ResponseEntity responseEntity = restTemplate.exchange(url, method, null, String.class); - - return objectMapper.readValue(responseEntity.getBody(), clazz); - } -} - -@Data -@NoArgsConstructor -@AllArgsConstructor -class FbPages { - private List data; - private Paging paging; - - @Data - static class Paging { - private String next; - } -} - -@Data -@NoArgsConstructor -@AllArgsConstructor -class FbLongLivingUserAccessToken { - private String accessToken; -} - -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -class FbApiError { - private String message; - private String type; - private Integer code; -} diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/facebook/FacebookSource.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/facebook/FacebookSource.java deleted file mode 100644 index d780eac4c4..0000000000 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/facebook/FacebookSource.java +++ /dev/null @@ -1,61 +0,0 @@ -package co.airy.core.api.admin.sources.facebook; - -import co.airy.core.api.admin.Source; -import co.airy.core.api.admin.SourceApiException; -import co.airy.core.api.admin.dto.ChannelMetadata; -import org.springframework.stereotype.Service; - -import java.util.List; - -import static java.util.stream.Collectors.toList; - -@Service -public class FacebookSource implements Source { - - private final FacebookApi api; - - public FacebookSource(FacebookApi api) { - this.api = api; - } - - public String getIdentifier() { - return "facebook"; - } - - public List getAvailableChannels(String token) throws SourceApiException { - try { - final List allPagesForUser = api.getAllPagesForUser(token); - - return allPagesForUser.stream() - .map((page) -> ChannelMetadata.builder() - .sourceChannelId(page.getId()) - .name(page.getNameWithLocationDescriptor()) - .imageUrl(page.getPicture().getData().getUrl()) - .build() - ).collect(toList()); - } catch (Exception e) { - throw new SourceApiException(e.getMessage()); - } - } - - public ChannelMetadata connectChannel(String token, String sourceChannelId) throws SourceApiException { - try { - final String longLivingUserToken = api.exchangeToLongLivingUserAccessToken(token); - final FbPageWithConnectInfo fbPageWithConnectInfo = api.getPageForUser(sourceChannelId, longLivingUserToken); - - api.connectPageToApp(fbPageWithConnectInfo.getAccessToken()); - - return ChannelMetadata.builder() - .name(fbPageWithConnectInfo.getNameWithLocationDescriptor()) - .sourceChannelId(fbPageWithConnectInfo.getId()) - .imageUrl(fbPageWithConnectInfo.getPicture().getData().getUrl()) - .build(); - } catch (Exception e) { - throw new SourceApiException(e.getMessage()); - } - } - - @Override - public void disconnectChannel(String token, String sourceChannelId) { - } -} diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/google/GoogleSource.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/google/GoogleSource.java deleted file mode 100644 index fabf870451..0000000000 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/google/GoogleSource.java +++ /dev/null @@ -1,29 +0,0 @@ -package co.airy.core.api.admin.sources.google; - -import co.airy.core.api.admin.Source; -import co.airy.core.api.admin.dto.ChannelMetadata; -import org.springframework.stereotype.Service; - -import java.util.List; - -@Service -public class GoogleSource implements Source { - @Override - public String getIdentifier() { - return "google"; - } - - @Override - public List getAvailableChannels(String token) { - return List.of(); - } - - @Override - public ChannelMetadata connectChannel(String token, String sourceChannelId) { - return new ChannelMetadata(); - } - - @Override - public void disconnectChannel(String token, String sourceChannelId) { - } -} diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/twilio/TwilioSmsSource.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/twilio/TwilioSmsSource.java deleted file mode 100644 index 1e2d8b5424..0000000000 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/twilio/TwilioSmsSource.java +++ /dev/null @@ -1,32 +0,0 @@ -package co.airy.core.api.admin.sources.twilio; - -import co.airy.core.api.admin.Source; -import co.airy.core.api.admin.dto.ChannelMetadata; -import org.springframework.stereotype.Service; - -import java.util.List; - -@Service -public class TwilioSmsSource implements Source { - @Override - public String getIdentifier() { - return "twilio.sms"; - } - - @Override - public List getAvailableChannels(String token) { - return List.of(); - } - - @Override - public ChannelMetadata connectChannel(String token, String sourceChannelId) { - return ChannelMetadata.builder() - .name("Twilio SMS Channel") - .sourceChannelId(sourceChannelId) - .build(); - } - - @Override - public void disconnectChannel(String token, String sourceChannelId) { - } -} diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/twilio/TwilioWhatsappSource.java b/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/twilio/TwilioWhatsappSource.java deleted file mode 100644 index d84b5bd087..0000000000 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/twilio/TwilioWhatsappSource.java +++ /dev/null @@ -1,32 +0,0 @@ -package co.airy.core.api.admin.sources.twilio; - -import co.airy.core.api.admin.Source; -import co.airy.core.api.admin.dto.ChannelMetadata; -import org.springframework.stereotype.Service; - -import java.util.List; - -@Service -public class TwilioWhatsappSource implements Source { - @Override - public String getIdentifier() { - return "twilio.whatsapp"; - } - - @Override - public List getAvailableChannels(String token) { - return List.of(); - } - - @Override - public ChannelMetadata connectChannel(String token, String sourceChannelId) { - return ChannelMetadata.builder() - .name("Twilio Whatsapp Channel") - .sourceChannelId(sourceChannelId) - .build(); - } - - @Override - public void disconnectChannel(String token, String sourceChannelId) { - } -} diff --git a/backend/api/admin/src/main/resources/application.properties b/backend/api/admin/src/main/resources/application.properties index 67b1c4388f..fbd90a5ed6 100644 --- a/backend/api/admin/src/main/resources/application.properties +++ b/backend/api/admin/src/main/resources/application.properties @@ -1,7 +1,7 @@ kafka.brokers=${KAFKA_BROKERS} kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} kafka.cleanup=${KAFKA_CLEANUP:false} -kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS:30000} +kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS} facebook.app-id=${FACEBOOK_APP_ID} facebook.app-secret=${FACEBOOK_APP_SECRET} diff --git a/backend/api/admin/src/test/java/co/airy/core/api/admin/ChannelsControllerTest.java b/backend/api/admin/src/test/java/co/airy/core/api/admin/ChannelsControllerTest.java index d36d4ca187..aab6d489e7 100644 --- a/backend/api/admin/src/test/java/co/airy/core/api/admin/ChannelsControllerTest.java +++ b/backend/api/admin/src/test/java/co/airy/core/api/admin/ChannelsControllerTest.java @@ -2,8 +2,6 @@ import co.airy.avro.communication.Channel; import co.airy.avro.communication.ChannelConnectionState; -import co.airy.core.api.admin.dto.ChannelMetadata; -import co.airy.core.api.admin.sources.facebook.FacebookSource; import co.airy.kafka.schema.application.ApplicationCommunicationChannels; import co.airy.kafka.schema.application.ApplicationCommunicationTags; import co.airy.kafka.schema.application.ApplicationCommunicationWebhooks; @@ -18,12 +16,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -32,11 +27,8 @@ import static co.airy.test.Timing.retryOnException; import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.not; -import static org.mockito.Mockito.doReturn; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -71,9 +63,6 @@ static void afterAll() throws Exception { kafkaTestHelper.afterAll(); } - @SpyBean - FacebookSource facebookSource; - private static boolean testDataInitialized = false; static final String facebookToken = "token"; @@ -88,8 +77,6 @@ static void afterAll() throws Exception { @BeforeEach void beforeEach() throws Exception { - MockitoAnnotations.initMocks(this); - if (testDataInitialized) { return; } @@ -124,74 +111,4 @@ void canListChannels() throws Exception { "/channels.list did not return the right number of channels"); } - @Test - void canExploreChannels() throws Exception { - final String channelName = "channel-name"; - - doReturn(List.of( - ChannelMetadata.builder() - .name(channelName) - .sourceChannelId("ps-id-1") - .build(), - ChannelMetadata.builder() - .sourceChannelId(connectedChannel.getSourceChannelId()) - .build() - )).when(facebookSource).getAvailableChannels(facebookToken); - - retryOnException(() -> webTestHelper.post("/channels.explore", - "{\"token\":\"" + facebookToken + "\",\"source\":\"facebook\"}", "user-id") - .andExpect(status().isOk()) - .andExpect(jsonPath("$.data", hasSize(2))) - .andExpect(jsonPath("$.data[0].name", equalTo(channelName))) - .andExpect(jsonPath("$.data[0].connected", equalTo(false))) - .andExpect(jsonPath("$.data[1].connected", equalTo(true))), - "/channels.list did not return the mocked channels"); - } - - - @Test - void canConnectChannel() throws Exception { - final String token = "token"; - final String channelName = "channel-name"; - final String sourceChannelId = "ps-id"; - - doReturn(new ChannelMetadata()).when(facebookSource).connectChannel(token, sourceChannelId); - - final String payload = "{\"token\":\"" + token + "\",\"source\":\"facebook\"," + - "\"source_channel_id\":\"" + sourceChannelId + "\"," + - "\"name\":\"" + channelName + "\"" + - "}"; - - retryOnException(() -> - webTestHelper.post("/channels.connect", payload, "user-id") - .andExpect(status().isOk()) - .andExpect(jsonPath("$.name", equalTo(channelName))) - .andExpect(jsonPath("$.source_channel_id", equalTo(sourceChannelId))), - "/channels.connect failed"); - } - - @Test - void disconnectChannel() throws Exception { - final String channelId = UUID.randomUUID().toString(); - - final Channel channel = Channel.newBuilder() - .setConnectionState(ChannelConnectionState.CONNECTED) - .setId(channelId) - .setName("connected channel name") - .setSource("facebook") - .setToken("disconnect-token") - .setSourceChannelId("disconnect-source-channel-id") - .build(); - - kafkaTestHelper.produceRecord(new ProducerRecord<>(applicationCommunicationChannels.name(), channelId, channel)); - - retryOnException(() -> - webTestHelper.post("/channels.disconnect", - "{\"channel_id\":\"" + channelId + "\"}", "user-id") - .andExpect(status().isOk()), - "/channels.disconnect failed"); - - Mockito.verify(facebookSource).disconnectChannel(channel.getToken(), channel.getSourceChannelId()); - } - } diff --git a/backend/api/auth/BUILD b/backend/api/auth/BUILD index 1e3f46c895..5d7896daa1 100644 --- a/backend/api/auth/BUILD +++ b/backend/api/auth/BUILD @@ -6,7 +6,6 @@ app_deps = [ "//backend:base_app", "//:springboot_actuator", "//:jdbi", - "//lib/java/payload", "//lib/java/spring/auth:spring-auth", "//lib/java/spring/web:spring-web", "//lib/java/pagination", @@ -32,6 +31,7 @@ springboot( junit5( size = "medium", file = file, + resources = glob(["src/test/resources/**/*"]), deps = [ ":app", "//backend:base_test", diff --git a/backend/api/auth/src/main/java/co/airy/core/api/auth/controllers/UsersController.java b/backend/api/auth/src/main/java/co/airy/core/api/auth/controllers/UsersController.java index 6fd2560836..153faf9aea 100644 --- a/backend/api/auth/src/main/java/co/airy/core/api/auth/controllers/UsersController.java +++ b/backend/api/auth/src/main/java/co/airy/core/api/auth/controllers/UsersController.java @@ -15,10 +15,10 @@ import co.airy.core.api.auth.dto.User; import co.airy.core.api.auth.services.Mail; import co.airy.core.api.auth.services.Password; -import co.airy.spring.jwt.Jwt; -import co.airy.payload.response.EmptyResponsePayload; -import co.airy.payload.response.RequestErrorResponsePayload; +import co.airy.spring.web.payload.EmptyResponsePayload; +import co.airy.spring.web.payload.RequestErrorResponsePayload; import co.airy.spring.auth.IgnoreAuthPattern; +import co.airy.spring.jwt.Jwt; import org.jdbi.v3.core.statement.UnableToExecuteStatementException; import org.springframework.context.annotation.Bean; import org.springframework.http.HttpStatus; diff --git a/backend/api/auth/src/test/java/co/airy/core/api/auth/UsersControllerTest.java b/backend/api/auth/src/test/java/co/airy/core/api/auth/UsersControllerTest.java index 92aecf3bf7..8568918987 100644 --- a/backend/api/auth/src/test/java/co/airy/core/api/auth/UsersControllerTest.java +++ b/backend/api/auth/src/test/java/co/airy/core/api/auth/UsersControllerTest.java @@ -5,8 +5,8 @@ import co.airy.core.api.auth.dao.UserDAO; import co.airy.core.api.auth.dto.User; import co.airy.core.api.auth.services.Mail; -import co.airy.spring.jwt.Jwt; import co.airy.spring.core.AirySpringBootApplication; +import co.airy.spring.jwt.Jwt; import co.airy.spring.test.WebTestHelper; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -38,19 +38,10 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @AutoConfigureEmbeddedDatabase(beanName = "dataSource") -@SpringBootTest(properties = { - "db.debug=true" -}, classes = AirySpringBootApplication.class) +@SpringBootTest(classes = AirySpringBootApplication.class) @AutoConfigureMockMvc @FlywayDataSource -@TestPropertySource(properties = { - "mail.host.url=localhost", - "mail.host.port=25", - "mail.sender.from=snasa@snasa.gov", - "mail.auth.username=snasa", - "mail.auth.password=extreme-secure-pass", - "auth.jwt-secret=this-needs-to-be-replaced-in-production-buffer:424242424242424242424242424242" -}) +@TestPropertySource(value = "classpath:test.properties") public class UsersControllerTest { @Autowired private ObjectMapper objectMapper; diff --git a/backend/api/auth/src/test/resources/test.properties b/backend/api/auth/src/test/resources/test.properties new file mode 100644 index 0000000000..a2ee4340ee --- /dev/null +++ b/backend/api/auth/src/test/resources/test.properties @@ -0,0 +1,6 @@ +mail.host.url=localhost +mail.host.port=25 +mail.sender.from=grace@example.com +mail.auth.username=ghopper +mail.auth.password=extreme-secure-pass +auth.jwt-secret=424242424242424242424242424242424242424242424242424242 \ No newline at end of file diff --git a/backend/api/communication/BUILD b/backend/api/communication/BUILD index d16eec3328..8e4097251c 100644 --- a/backend/api/communication/BUILD +++ b/backend/api/communication/BUILD @@ -7,12 +7,12 @@ app_deps = [ "//backend:base_app", "//:springboot_actuator", "//:springboot_websocket", - "//backend:message", - "//backend:channel", - "//backend:metadata", + "//backend/model/message:message", + "//backend/model/channel:channel", + "//backend/model/metadata:metadata", "//backend:read-receipt", "//lib/java/mapping", - "//lib/java/payload", + "//lib/java/date", "//lib/java/pagination", "//lib/java/spring/auth:spring-auth", "//lib/java/spring/web:spring-web", diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/ConversationsController.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/ConversationsController.java index f72eb53dcc..eb9fb1e79d 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/ConversationsController.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/ConversationsController.java @@ -1,8 +1,8 @@ package co.airy.core.api.communication; -import co.airy.avro.communication.MetadataAction; -import co.airy.avro.communication.MetadataActionType; -import co.airy.avro.communication.MetadataKeys; +import co.airy.avro.communication.Metadata; +import co.airy.model.metadata.MetadataKeys; +import co.airy.model.metadata.Subject; import co.airy.avro.communication.ReadReceipt; import co.airy.core.api.communication.dto.Conversation; import co.airy.core.api.communication.dto.ConversationIndex; @@ -16,7 +16,7 @@ import co.airy.core.api.communication.payload.ResponseMetadata; import co.airy.pagination.Page; import co.airy.pagination.Paginator; -import co.airy.payload.response.RequestErrorResponsePayload; +import co.airy.spring.web.payload.RequestErrorResponsePayload; import org.apache.kafka.streams.state.KeyValueIterator; import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.List; +import static co.airy.model.metadata.MetadataRepository.newConversationTag; import static java.util.Comparator.comparing; import static java.util.stream.Collectors.toList; @@ -57,7 +58,7 @@ ResponseEntity conversationList(@RequestBody @Valid ConversationListRequestPa return queryConversations(requestPayload); } - private ResponseEntity queryConversations(ConversationListRequestPayload requestPayload) throws Exception { + private ResponseEntity queryConversations(ConversationListRequestPayload requestPayload) { final ReadOnlyLuceneStore conversationLuceneStore = stores.getConversationLuceneStore(); final ReadOnlyKeyValueStore conversationsStore = stores.getConversationsStore(); @@ -179,15 +180,28 @@ ResponseEntity conversationMarkRead(@RequestBody @Valid ConversationByIdReque @PostMapping("/conversations.tag") ResponseEntity conversationTag(@RequestBody @Valid ConversationTagRequestPayload requestPayload) { - return setConversationTag(requestPayload, MetadataActionType.SET); + final String conversationId = requestPayload.getConversationId().toString(); + final String tagId = requestPayload.getTagId().toString(); + final ReadOnlyKeyValueStore store = stores.getConversationsStore(); + final Conversation conversation = store.get(conversationId); + + if (conversation == null) { + return ResponseEntity.notFound().build(); + } + + final Metadata metadata = newConversationTag(conversationId, tagId); + + try { + stores.storeMetadata(metadata); + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new RequestErrorResponsePayload(e.getMessage())); + } + + return ResponseEntity.accepted().build(); } @PostMapping("/conversations.untag") ResponseEntity conversationUntag(@RequestBody @Valid ConversationTagRequestPayload requestPayload) { - return setConversationTag(requestPayload, MetadataActionType.REMOVE); - } - - private ResponseEntity setConversationTag(ConversationTagRequestPayload requestPayload, MetadataActionType actionType) { final String conversationId = requestPayload.getConversationId().toString(); final String tagId = requestPayload.getTagId().toString(); final ReadOnlyKeyValueStore store = stores.getConversationsStore(); @@ -197,16 +211,10 @@ private ResponseEntity setConversationTag(ConversationTagRequestPayload reque return ResponseEntity.notFound().build(); } - final MetadataAction metadataAction = MetadataAction.newBuilder() - .setActionType(actionType) - .setTimestamp(Instant.now().toEpochMilli()) - .setConversationId(conversationId) - .setValue("") - .setKey(String.format("%s.%s", MetadataKeys.TAGS, tagId)) - .build(); - try { - stores.storeMetadata(metadataAction); + final Subject subject = new Subject("conversation", conversationId); + final String metadataKey = String.format("%s.%s", MetadataKeys.TAGS, tagId); + stores.deleteMetadata(subject, metadataKey); } catch (Exception e) { return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new RequestErrorResponsePayload(e.getMessage())); } diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/Mapper.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/Mapper.java index d2508360f9..a178e4e3b9 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/Mapper.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/Mapper.java @@ -1,22 +1,22 @@ package co.airy.core.api.communication; import co.airy.avro.communication.Message; -import co.airy.avro.communication.MetadataKeys; -import co.airy.avro.communication.MetadataMapper; +import co.airy.model.metadata.MetadataKeys; +import co.airy.model.metadata.MetadataRepository; +import co.airy.model.channel.ChannelPayload; import co.airy.core.api.communication.dto.Conversation; import co.airy.core.api.communication.dto.DisplayName; import co.airy.core.api.communication.payload.ContactResponsePayload; import co.airy.core.api.communication.payload.ConversationResponsePayload; import co.airy.core.api.communication.payload.MessageResponsePayload; import co.airy.mapping.ContentMapper; -import co.airy.payload.response.ChannelPayload; import org.springframework.stereotype.Component; import java.util.Map; -import static co.airy.avro.communication.MetadataKeys.PUBLIC; -import static co.airy.avro.communication.MetadataMapper.filterPrefix; -import static co.airy.payload.format.DateFormat.isoFromMillis; +import static co.airy.model.metadata.MetadataRepository.getConversationInfo; +import static co.airy.date.format.DateFormat.isoFromMillis; +import static java.util.stream.Collectors.toList; @Component public class Mapper { @@ -33,10 +33,17 @@ public ConversationResponsePayload fromConversation(Conversation conversation) { .channel(ChannelPayload.builder() .id(conversation.getChannelId()) .name(conversation.getChannel().getName()) + .source(conversation.getChannel().getSource()) .build()) .id(conversation.getId()) .unreadMessageCount(conversation.getUnreadCount()) - .tags(MetadataMapper.getTags(metadata)) + .tags( + MetadataRepository.filterPrefix(metadata, MetadataKeys.TAGS) + .keySet() + .stream() + .map(s -> s.split("\\.")[1]) + .collect(toList()) + ) .createdAt(isoFromMillis(conversation.getCreatedAt())) .contact(getContact(conversation)) .lastMessage(fromMessage(conversation.getLastMessage())) @@ -51,7 +58,7 @@ private ContactResponsePayload getContact(Conversation conversation) { .avatarUrl(metadata.get(MetadataKeys.Source.Contact.AVATAR_URL)) .firstName(displayName.getFirstName()) .lastName(displayName.getLastName()) - .info(filterPrefix(metadata, PUBLIC)) + .info(getConversationInfo(metadata)) .build(); } diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/MetadataController.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/MetadataController.java index de36a1d794..8ac527219b 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/MetadataController.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/MetadataController.java @@ -1,11 +1,11 @@ package co.airy.core.api.communication; -import co.airy.avro.communication.MetadataAction; -import co.airy.avro.communication.MetadataActionType; +import co.airy.avro.communication.Metadata; +import co.airy.model.metadata.Subject; import co.airy.core.api.communication.payload.RemoveMetadataRequestPayload; import co.airy.core.api.communication.payload.SetMetadataRequestPayload; -import co.airy.payload.response.EmptyResponsePayload; -import co.airy.payload.response.RequestErrorResponsePayload; +import co.airy.spring.web.payload.EmptyResponsePayload; +import co.airy.spring.web.payload.RequestErrorResponsePayload; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; @@ -13,9 +13,9 @@ import org.springframework.web.bind.annotation.RestController; import javax.validation.Valid; -import java.time.Instant; -import static co.airy.avro.communication.MetadataKeys.PUBLIC; +import static co.airy.model.metadata.MetadataKeys.PUBLIC; +import static co.airy.model.metadata.MetadataRepository.newConversationMetadata; @RestController public class MetadataController { @@ -26,16 +26,12 @@ public MetadataController(Stores stores) { } @PostMapping("/metadata.set") - ResponseEntity setMetadata(@RequestBody @Valid SetMetadataRequestPayload setMetadataRequestPayload) { - final MetadataAction metadataAction = MetadataAction.newBuilder() - .setActionType(MetadataActionType.SET) - .setTimestamp(Instant.now().toEpochMilli()) - .setConversationId(setMetadataRequestPayload.getConversationId()) - .setValue(setMetadataRequestPayload.getValue()) - .setKey(PUBLIC + "." + setMetadataRequestPayload.getKey()) - .build(); + ResponseEntity setMetadata(@RequestBody @Valid SetMetadataRequestPayload requestPayload) { + final Metadata metadata = newConversationMetadata(requestPayload.getConversationId(), + PUBLIC + "." + requestPayload.getKey(), + requestPayload.getValue()); try { - stores.storeMetadata(metadataAction); + stores.storeMetadata(metadata); } catch (Exception e) { return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new RequestErrorResponsePayload(e.getMessage())); } @@ -43,16 +39,12 @@ ResponseEntity setMetadata(@RequestBody @Valid SetMetadataRequestPayload setM } @PostMapping("/metadata.remove") - ResponseEntity removeMetadata(@RequestBody @Valid RemoveMetadataRequestPayload removeMetadataRequestPayload) { - final MetadataAction metadataAction = MetadataAction.newBuilder() - .setActionType(MetadataActionType.REMOVE) - .setTimestamp(Instant.now().toEpochMilli()) - .setConversationId(removeMetadataRequestPayload.getConversationId()) - .setKey(PUBLIC + "." + removeMetadataRequestPayload.getKey()) - .setValue("") - .build(); + ResponseEntity removeMetadata(@RequestBody @Valid RemoveMetadataRequestPayload requestPayload) { + final Subject subject = new Subject("conversation", requestPayload.getConversationId()); + final String metadataKey = PUBLIC + "." + requestPayload.getKey(); + try { - stores.storeMetadata(metadataAction); + stores.deleteMetadata(subject, metadataKey); } catch (Exception e) { return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new RequestErrorResponsePayload(e.getMessage())); } diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/SendMessageController.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/SendMessageController.java index 364eeaead7..572d415816 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/SendMessageController.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/SendMessageController.java @@ -8,7 +8,7 @@ import co.airy.core.api.communication.dto.Conversation; import co.airy.core.api.communication.payload.SendMessageRequestPayload; import co.airy.kafka.schema.application.ApplicationCommunicationMessages; -import co.airy.payload.response.EmptyResponsePayload; +import co.airy.spring.web.payload.EmptyResponsePayload; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.kafka.clients.producer.KafkaProducer; diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/Stores.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/Stores.java index e2c7357142..f96d48e0cd 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/Stores.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/Stores.java @@ -2,8 +2,7 @@ import co.airy.avro.communication.Channel; import co.airy.avro.communication.Message; -import co.airy.avro.communication.MetadataAction; -import co.airy.avro.communication.MetadataActionType; +import co.airy.avro.communication.Metadata; import co.airy.avro.communication.ReadReceipt; import co.airy.avro.communication.SenderType; import co.airy.core.api.communication.dto.Conversation; @@ -19,9 +18,11 @@ import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; import co.airy.kafka.schema.application.ApplicationCommunicationReadReceipts; import co.airy.kafka.streams.KafkaStreamsWrapper; +import co.airy.model.metadata.Subject; import org.apache.avro.specific.SpecificRecordBase; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.kstream.KGroupedStream; import org.apache.kafka.streams.kstream.KStream; @@ -44,6 +45,9 @@ import java.util.Map; import java.util.concurrent.ExecutionException; +import static co.airy.model.metadata.MetadataRepository.getId; +import static co.airy.model.metadata.MetadataRepository.getSubject; +import static co.airy.model.metadata.MetadataRepository.isConversationMetadata; import static java.util.stream.Collectors.toCollection; @Component @@ -86,15 +90,14 @@ private void startStream() { .peek((channelId, channel) -> webSocketController.onChannelUpdate(channel)) .toTable(); - final KTable> metadataTable = builder.stream(applicationCommunicationMetadata) - .groupByKey() - .aggregate(HashMap::new, (conversationId, metadataAction, aggregate) -> { - if (metadataAction.getActionType().equals(MetadataActionType.SET)) { - aggregate.put(metadataAction.getKey(), metadataAction.getValue()); - } else { - aggregate.remove(metadataAction.getKey()); - } - + final KTable> metadataTable = builder.table(applicationCommunicationMetadata) + .filter((metadataId, metadata) -> isConversationMetadata(metadata)) + .groupBy((metadataId, metadata) -> KeyValue.pair(getSubject(metadata).getIdentifier(), metadata)) + .aggregate(HashMap::new, (conversationId, metadata, aggregate) -> { + aggregate.put(metadata.getKey(), metadata.getValue()); + return aggregate; + }, (conversationId, metadata, aggregate) -> { + aggregate.remove(metadata.getKey()); return aggregate; }); @@ -192,8 +195,12 @@ public void storeReadReceipt(ReadReceipt readReceipt) throws ExecutionException, producer.send(new ProducerRecord<>(applicationCommunicationReadReceipts, readReceipt.getConversationId(), readReceipt)).get(); } - public void storeMetadata(MetadataAction metadataAction) throws ExecutionException, InterruptedException { - producer.send(new ProducerRecord<>(applicationCommunicationMetadata, metadataAction.getConversationId(), metadataAction)).get(); + public void storeMetadata(Metadata metadata) throws ExecutionException, InterruptedException { + producer.send(new ProducerRecord<>(applicationCommunicationMetadata, getId(metadata).toString(), metadata)).get(); + } + + public void deleteMetadata(Subject subject, String key) throws ExecutionException, InterruptedException { + producer.send(new ProducerRecord<>(applicationCommunicationMetadata, getId(subject, key).toString(), null)).get(); } public List getMessages(String conversationId) { diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/WebSocketConfig.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/WebSocketConfig.java index 200d03abbb..2ad9c99ee0 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/WebSocketConfig.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/WebSocketConfig.java @@ -1,7 +1,7 @@ package co.airy.core.api.communication; -import co.airy.spring.jwt.Jwt; import co.airy.log.AiryLoggerFactory; +import co.airy.spring.jwt.Jwt; import org.slf4j.Logger; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/WebSocketController.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/WebSocketController.java index 67a80e2ad0..5df863ce0c 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/WebSocketController.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/WebSocketController.java @@ -1,18 +1,19 @@ package co.airy.core.api.communication; import co.airy.avro.communication.Channel; -import co.airy.avro.communication.ChannelConnectionState; import co.airy.avro.communication.Message; +import co.airy.model.channel.ChannelPayload; import co.airy.core.api.communication.dto.UnreadCountState; import co.airy.core.api.communication.payload.MessageUpsertPayload; import co.airy.core.api.communication.payload.UnreadCountPayload; -import co.airy.payload.response.ChannelPayload; import org.springframework.messaging.simp.SimpMessagingTemplate; import org.springframework.stereotype.Service; import java.time.Instant; -import static co.airy.payload.format.DateFormat.isoFromMillis; +import static co.airy.avro.communication.ChannelConnectionState.CONNECTED; +import static co.airy.model.channel.ChannelPayload.fromChannel; +import static co.airy.date.format.DateFormat.isoFromMillis; @Service public class WebSocketController { @@ -49,18 +50,11 @@ public void onUnreadCount(String conversationId, UnreadCountState unreadCountSta } public void onChannelUpdate(Channel channel) { - final ChannelPayload channelPayload = ChannelPayload.builder() - .imageUrl(channel.getImageUrl()) - .source(channel.getSource()) - .sourceChannelId(channel.getSourceChannelId()) - .name(channel.getName()) - .id(channel.getId()) - .build(); + final ChannelPayload channelPayload = fromChannel(channel); + final String queue = CONNECTED.equals(channel.getConnectionState()) ? + QUEUE_CHANNEL_CONNECTED : + QUEUE_CHANNEL_DISCONNECTED; - if (ChannelConnectionState.CONNECTED.equals(channel.getConnectionState())) { - messagingTemplate.convertAndSend(QUEUE_CHANNEL_CONNECTED, channelPayload); - } else { - messagingTemplate.convertAndSend(QUEUE_CHANNEL_DISCONNECTED, channelPayload); - } + messagingTemplate.convertAndSend(queue, channelPayload); } } diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/dto/Conversation.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/dto/Conversation.java index b93330bdf4..31f85edadf 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/dto/Conversation.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/dto/Conversation.java @@ -2,8 +2,7 @@ import co.airy.avro.communication.Channel; import co.airy.avro.communication.Message; -import co.airy.avro.communication.MetadataKeys; -import co.airy.core.api.communication.payload.ContactResponsePayload; +import co.airy.model.metadata.MetadataKeys; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Builder; @@ -14,8 +13,6 @@ import java.util.HashMap; import java.util.Map; -import static co.airy.avro.communication.MetadataKeys.PUBLIC; -import static co.airy.avro.communication.MetadataMapper.filterPrefix; import static org.springframework.util.StringUtils.capitalize; @Data diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/dto/ConversationIndex.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/dto/ConversationIndex.java index 41a547336e..d766746c81 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/dto/ConversationIndex.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/dto/ConversationIndex.java @@ -17,6 +17,7 @@ public class ConversationIndex implements Serializable { private String id; private String displayName; private String channelId; + private String source; private Long createdAt; private Integer unreadCount; @@ -27,6 +28,7 @@ public static ConversationIndex fromConversation(Conversation conversation) { return ConversationIndex.builder() .id(conversation.getId()) .channelId(conversation.getChannelId()) + .source(conversation.getChannel().getSource()) .displayName(conversation.getDisplayNameOrDefault().toString()) .metadata(new HashMap<>(conversation.getMetadata())) .createdAt(conversation.getCreatedAt()) diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/DocumentMapper.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/DocumentMapper.java index 3f8fe25f1f..c0a0b057ec 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/DocumentMapper.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/DocumentMapper.java @@ -11,7 +11,6 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexableField; -import java.io.IOException; import java.util.Map; import static java.util.stream.Collectors.toMap; diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/LuceneDiskStore.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/LuceneDiskStore.java index bf37839df3..c05522c093 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/LuceneDiskStore.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/LuceneDiskStore.java @@ -3,12 +3,8 @@ import co.airy.core.api.communication.dto.ConversationIndex; import co.airy.core.api.communication.dto.LuceneQueryResult; import co.airy.kafka.core.serdes.KafkaHybridSerde; -import co.airy.kafka.core.serializer.KafkaJacksonSerializer; import co.airy.log.AiryLoggerFactory; import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.common.utils.Bytes; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.processor.AbstractNotifyingBatchingRestoreCallback; import org.apache.kafka.streams.processor.ProcessorContext; import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.internals.ProcessorStateManager; @@ -20,7 +16,6 @@ import java.io.IOException; import java.io.Serializable; -import java.util.Collection; import java.util.Map; public class LuceneDiskStore implements StateStore, LuceneStore { diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/LuceneProvider.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/LuceneProvider.java index 3d5e0588ed..9b3835011d 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/LuceneProvider.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/LuceneProvider.java @@ -3,7 +3,6 @@ import co.airy.core.api.communication.dto.ConversationIndex; import co.airy.core.api.communication.dto.LuceneQueryResult; import co.airy.log.AiryLoggerFactory; -import org.apache.kafka.streams.KeyValue; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; @@ -21,11 +20,8 @@ import java.io.IOException; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import static java.util.stream.Collectors.toList; - @Component public class LuceneProvider implements LuceneStore { private static final Logger log = AiryLoggerFactory.getLogger(LuceneDiskStore.class); diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/payload/ConversationListRequestPayload.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/payload/ConversationListRequestPayload.java index 1173530c46..9ad6ed2634 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/payload/ConversationListRequestPayload.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/payload/ConversationListRequestPayload.java @@ -10,5 +10,5 @@ public class ConversationListRequestPayload { private String filters; private String cursor; - private int pageSize = 10; + private int pageSize = 20; } diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/payload/ConversationResponsePayload.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/payload/ConversationResponsePayload.java index 8d2e9bfbd7..a4b4acb3cb 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/payload/ConversationResponsePayload.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/payload/ConversationResponsePayload.java @@ -1,6 +1,6 @@ package co.airy.core.api.communication.payload; -import co.airy.payload.response.ChannelPayload; +import co.airy.model.channel.ChannelPayload; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; diff --git a/backend/api/communication/src/main/java/co/airy/core/api/communication/payload/MessageResponsePayload.java b/backend/api/communication/src/main/java/co/airy/core/api/communication/payload/MessageResponsePayload.java index 00e34b7ca9..806db54106 100644 --- a/backend/api/communication/src/main/java/co/airy/core/api/communication/payload/MessageResponsePayload.java +++ b/backend/api/communication/src/main/java/co/airy/core/api/communication/payload/MessageResponsePayload.java @@ -1,19 +1,20 @@ package co.airy.core.api.communication.payload; -import co.airy.avro.communication.SenderType; import co.airy.mapping.model.Content; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + @Data @Builder @NoArgsConstructor @AllArgsConstructor public class MessageResponsePayload { private String id; - private Content content; + private List content; private String senderType; private String sentAt; private String deliveryState; diff --git a/backend/api/communication/src/main/resources/application.properties b/backend/api/communication/src/main/resources/application.properties index b1074b0e7d..cba3b56051 100644 --- a/backend/api/communication/src/main/resources/application.properties +++ b/backend/api/communication/src/main/resources/application.properties @@ -1,6 +1,6 @@ kafka.brokers=${KAFKA_BROKERS} kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} kafka.cleanup=${KAFKA_CLEANUP:false} -kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS:30000} +kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS} auth.jwt-secret=${JWT_SECRET} diff --git a/backend/api/communication/src/test/java/co/airy/core/api/communication/ConversationsListTest.java b/backend/api/communication/src/test/java/co/airy/core/api/communication/ConversationsListTest.java index 7176a7c6df..1f486ff277 100644 --- a/backend/api/communication/src/test/java/co/airy/core/api/communication/ConversationsListTest.java +++ b/backend/api/communication/src/test/java/co/airy/core/api/communication/ConversationsListTest.java @@ -2,7 +2,7 @@ import co.airy.avro.communication.Channel; import co.airy.avro.communication.ChannelConnectionState; -import co.airy.avro.communication.MetadataKeys; +import co.airy.model.metadata.MetadataKeys; import co.airy.core.api.communication.util.TestConversation; import co.airy.kafka.schema.application.ApplicationCommunicationChannels; import co.airy.kafka.schema.application.ApplicationCommunicationMessages; @@ -10,7 +10,7 @@ import co.airy.kafka.schema.application.ApplicationCommunicationReadReceipts; import co.airy.kafka.test.KafkaTestHelper; import co.airy.kafka.test.junit.SharedKafkaTestResource; -import co.airy.payload.format.DateFormat; +import co.airy.date.format.DateFormat; import co.airy.spring.core.AirySpringBootApplication; import co.airy.spring.test.WebTestHelper; import com.fasterxml.jackson.databind.node.JsonNodeFactory; diff --git a/backend/api/communication/src/test/java/co/airy/core/api/communication/MessagesTest.java b/backend/api/communication/src/test/java/co/airy/core/api/communication/MessagesTest.java index 9705d7e760..c2f09d46a4 100644 --- a/backend/api/communication/src/test/java/co/airy/core/api/communication/MessagesTest.java +++ b/backend/api/communication/src/test/java/co/airy/core/api/communication/MessagesTest.java @@ -10,7 +10,7 @@ import co.airy.kafka.schema.application.ApplicationCommunicationReadReceipts; import co.airy.kafka.test.KafkaTestHelper; import co.airy.kafka.test.junit.SharedKafkaTestResource; -import co.airy.payload.format.DateFormat; +import co.airy.date.format.DateFormat; import co.airy.spring.core.AirySpringBootApplication; import co.airy.spring.test.WebTestHelper; import org.apache.avro.specific.SpecificRecordBase; diff --git a/backend/api/communication/src/test/java/co/airy/core/api/communication/MetadataControllerTest.java b/backend/api/communication/src/test/java/co/airy/core/api/communication/MetadataControllerTest.java index cb5304afad..daa4800552 100644 --- a/backend/api/communication/src/test/java/co/airy/core/api/communication/MetadataControllerTest.java +++ b/backend/api/communication/src/test/java/co/airy/core/api/communication/MetadataControllerTest.java @@ -27,8 +27,6 @@ import java.util.UUID; import static co.airy.test.Timing.retryOnException; -import static org.hamcrest.core.Is.is; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = AirySpringBootApplication.class) diff --git a/backend/api/communication/src/test/java/co/airy/core/api/communication/WebSocketControllerTest.java b/backend/api/communication/src/test/java/co/airy/core/api/communication/WebSocketControllerTest.java index 45842d0971..685dcfd57f 100644 --- a/backend/api/communication/src/test/java/co/airy/core/api/communication/WebSocketControllerTest.java +++ b/backend/api/communication/src/test/java/co/airy/core/api/communication/WebSocketControllerTest.java @@ -2,18 +2,18 @@ import co.airy.avro.communication.Channel; import co.airy.avro.communication.ChannelConnectionState; +import co.airy.model.channel.ChannelPayload; import co.airy.core.api.communication.payload.MessageUpsertPayload; import co.airy.core.api.communication.payload.UnreadCountPayload; import co.airy.core.api.communication.util.TestConversation; -import co.airy.spring.jwt.Jwt; import co.airy.kafka.schema.application.ApplicationCommunicationChannels; import co.airy.kafka.schema.application.ApplicationCommunicationMessages; import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; import co.airy.kafka.schema.application.ApplicationCommunicationReadReceipts; import co.airy.kafka.test.KafkaTestHelper; import co.airy.kafka.test.junit.SharedKafkaTestResource; -import co.airy.payload.response.ChannelPayload; import co.airy.spring.core.AirySpringBootApplication; +import co.airy.spring.jwt.Jwt; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; import org.apache.kafka.clients.producer.ProducerRecord; diff --git a/backend/api/communication/src/test/java/co/airy/core/api/communication/util/TestConversation.java b/backend/api/communication/src/test/java/co/airy/core/api/communication/util/TestConversation.java index 14b6530eb6..7932d61c51 100644 --- a/backend/api/communication/src/test/java/co/airy/core/api/communication/util/TestConversation.java +++ b/backend/api/communication/src/test/java/co/airy/core/api/communication/util/TestConversation.java @@ -3,8 +3,6 @@ import co.airy.avro.communication.Channel; import co.airy.avro.communication.DeliveryState; import co.airy.avro.communication.Message; -import co.airy.avro.communication.MetadataAction; -import co.airy.avro.communication.MetadataActionType; import co.airy.avro.communication.SenderType; import co.airy.kafka.schema.application.ApplicationCommunicationMessages; import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; @@ -21,6 +19,8 @@ import java.util.Random; import java.util.UUID; +import static co.airy.model.metadata.MetadataRepository.newConversationMetadata; + @Data @NoArgsConstructor public class TestConversation { @@ -67,13 +67,7 @@ private List> generateRecords() { if (metadata != null) { metadata.forEach((metadataKey, metadataValue) -> records.add(new ProducerRecord<>(applicationCommunicationMetadata, conversationId, - MetadataAction.newBuilder() - .setKey(metadataKey) - .setValue(metadataValue) - .setConversationId(conversationId) - .setActionType(MetadataActionType.SET) - .setTimestamp(Instant.now().toEpochMilli()) - .build() + newConversationMetadata(conversationId, metadataKey, metadataValue) ))); } diff --git a/backend/avro/communication/BUILD b/backend/avro/communication/BUILD index 67ab81f543..15191666b0 100644 --- a/backend/avro/communication/BUILD +++ b/backend/avro/communication/BUILD @@ -1,10 +1,10 @@ load("//tools/build:avro.bzl", "avro_java_library") -load("//tools/build:java_library.bzl", "custom_java_library") package(default_visibility = ["//visibility:public"]) avro_java_library( - name = "channel", + name = "channel-avro", + srcs = ["channel.avsc"], ) avro_java_library( @@ -15,30 +15,14 @@ avro_java_library( name = "read-receipt", ) -custom_java_library( - name = "message", - srcs = ["message/src/main/java/co/airy/avro/communication/MessageRepository.java"], - exports = [":message-avro"], - deps = [":message-avro"], -) - avro_java_library( name = "message-avro", srcs = ["message.avsc"], - visibility = ["//visibility:private"], -) - -custom_java_library( - name = "metadata-action", - srcs = glob(["metadata-action/src/main/**/*.java"]), - exports = [":metadata-action-avro"], - deps = [":metadata-action-avro"], ) avro_java_library( - name = "metadata-action-avro", - srcs = ["metadata-action.avsc"], - visibility = ["//visibility:private"], + name = "metadata-avro", + srcs = ["metadata.avsc"], ) avro_java_library( diff --git a/backend/avro/communication/metadata-action/src/main/java/co/airy/avro/communication/MetadataMapper.java b/backend/avro/communication/metadata-action/src/main/java/co/airy/avro/communication/MetadataMapper.java deleted file mode 100644 index d665212a91..0000000000 --- a/backend/avro/communication/metadata-action/src/main/java/co/airy/avro/communication/MetadataMapper.java +++ /dev/null @@ -1,26 +0,0 @@ -package co.airy.avro.communication; - -import java.util.List; -import java.util.Map; - -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; - -public class MetadataMapper { - public static Map filterPrefix(Map metadataMap, String prefix) { - return metadataMap - .entrySet() - .stream() - .filter((entry) -> entry.getKey().startsWith(prefix)) - .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - public static List getTags(Map metadataMap) { - return metadataMap - .keySet() - .stream() - .filter(s -> s.startsWith(MetadataKeys.TAGS)) - .map(s -> s.split("\\.")[1]) - .collect(toList()); - } -} diff --git a/backend/avro/communication/metadata-action.avsc b/backend/avro/communication/metadata.avsc similarity index 55% rename from backend/avro/communication/metadata-action.avsc rename to backend/avro/communication/metadata.avsc index 60aab9b926..86d73cd134 100644 --- a/backend/avro/communication/metadata-action.avsc +++ b/backend/avro/communication/metadata.avsc @@ -1,12 +1,11 @@ { "namespace": "co.airy.avro.communication", - "name": "MetadataAction", + "name": "Metadata", "type": "record", "fields": [ - {"name": "conversationId", "type": "string"}, + {"name": "subject", "type": "string"}, {"name": "key", "type": "string"}, {"name": "value", "type": "string"}, - {"name": "actionType", "type": {"name": "MetadataActionType", "type": "enum", "symbols": ["SET", "REMOVE"]}}, {"name": "timestamp", "type": "long", "logicalType": "timestamp-millis"} ] } diff --git a/backend/model/channel/BUILD b/backend/model/channel/BUILD new file mode 100644 index 0000000000..ce160a468e --- /dev/null +++ b/backend/model/channel/BUILD @@ -0,0 +1,16 @@ +load("//tools/build:java_library.bzl", "custom_java_library") + +custom_java_library( + name = "channel", + srcs = glob(["src/main/java/co/airy/model/channel/**/*.java"]), + visibility = ["//visibility:public"], + exports = [ + "//backend/avro/communication:channel-avro", + "//lib/java/kafka/schema:application-communication-channels", + ], + deps = [ + "//:jackson", + "//:lombok", + "//backend/avro/communication:channel-avro", + ], +) diff --git a/lib/java/payload/src/main/java/co/airy/payload/response/ChannelPayload.java b/backend/model/channel/src/main/java/co/airy/model/channel/ChannelPayload.java similarity index 56% rename from lib/java/payload/src/main/java/co/airy/payload/response/ChannelPayload.java rename to backend/model/channel/src/main/java/co/airy/model/channel/ChannelPayload.java index fa30148229..72a98faaf3 100644 --- a/lib/java/payload/src/main/java/co/airy/payload/response/ChannelPayload.java +++ b/backend/model/channel/src/main/java/co/airy/model/channel/ChannelPayload.java @@ -1,5 +1,6 @@ -package co.airy.payload.response; +package co.airy.model.channel; +import co.airy.avro.communication.Channel; import com.fasterxml.jackson.annotation.JsonInclude; import lombok.AllArgsConstructor; import lombok.Builder; @@ -26,4 +27,14 @@ public class ChannelPayload { @JsonInclude(NON_NULL) private String imageUrl; + + public static ChannelPayload fromChannel(Channel channel) { + return ChannelPayload.builder() + .name(channel.getName()) + .id(channel.getId()) + .imageUrl(channel.getImageUrl()) + .source(channel.getSource()) + .sourceChannelId(channel.getSourceChannelId()) + .build(); + } } \ No newline at end of file diff --git a/backend/model/message/BUILD b/backend/model/message/BUILD new file mode 100644 index 0000000000..0014d90336 --- /dev/null +++ b/backend/model/message/BUILD @@ -0,0 +1,14 @@ +load("//tools/build:java_library.bzl", "custom_java_library") + +custom_java_library( + name = "message", + srcs = glob(["src/main/java/co/airy/model/message/**/*.java"]), + visibility = ["//visibility:public"], + exports = [ + "//backend/avro/communication:message-avro", + "//lib/java/kafka/schema:application-communication-messages", + ], + deps = [ + "//backend/avro/communication:message-avro", + ], +) diff --git a/backend/avro/communication/message/src/main/java/co/airy/avro/communication/MessageRepository.java b/backend/model/message/src/main/java/co/airy/model/message/MessageRepository.java similarity index 69% rename from backend/avro/communication/message/src/main/java/co/airy/avro/communication/MessageRepository.java rename to backend/model/message/src/main/java/co/airy/model/message/MessageRepository.java index b1a7129284..fa9cabe23d 100644 --- a/backend/avro/communication/message/src/main/java/co/airy/avro/communication/MessageRepository.java +++ b/backend/model/message/src/main/java/co/airy/model/message/MessageRepository.java @@ -1,4 +1,7 @@ -package co.airy.avro.communication; +package co.airy.model.message; + +import co.airy.avro.communication.DeliveryState; +import co.airy.avro.communication.Message; import java.time.Instant; diff --git a/backend/model/metadata/BUILD b/backend/model/metadata/BUILD new file mode 100644 index 0000000000..dca2843637 --- /dev/null +++ b/backend/model/metadata/BUILD @@ -0,0 +1,16 @@ +load("//tools/build:java_library.bzl", "custom_java_library") + +custom_java_library( + name = "metadata", + srcs = glob(["src/main/java/co/airy/model/metadata/**/*.java"]), + visibility = ["//visibility:public"], + exports = [ + "//backend/avro/communication:metadata-avro", + "//lib/java/kafka/schema:application-communication-metadata", + ], + deps = [ + "//:lombok", + "//backend/avro/communication:metadata-avro", + "//lib/java/uuid", + ], +) diff --git a/backend/avro/communication/metadata-action/src/main/java/co/airy/avro/communication/MetadataKeys.java b/backend/model/metadata/src/main/java/co/airy/model/metadata/MetadataKeys.java similarity index 52% rename from backend/avro/communication/metadata-action/src/main/java/co/airy/avro/communication/MetadataKeys.java rename to backend/model/metadata/src/main/java/co/airy/model/metadata/MetadataKeys.java index 7bab6258da..dc10ba3ba9 100644 --- a/backend/avro/communication/metadata-action/src/main/java/co/airy/avro/communication/MetadataKeys.java +++ b/backend/model/metadata/src/main/java/co/airy/model/metadata/MetadataKeys.java @@ -1,4 +1,4 @@ -package co.airy.avro.communication; +package co.airy.model.metadata; /** * JSON dot notation keys for pre-defined metadata @@ -10,6 +10,23 @@ public static class Contact { public static final String FIRST_NAME = "source.contact.first_name"; public static final String LAST_NAME = "source.contact.last_name"; public static final String AVATAR_URL = "source.contact.avatar_url"; + public static final String FETCH_STATE = "source.contact.fetch_state"; + } + + public enum ContactFetchState { + ok("ok"), + failed("failed"); + + private final String state; + + ContactFetchState(final String state) { + this.state = state; + } + + @Override + public String toString() { + return state; + } } } diff --git a/backend/model/metadata/src/main/java/co/airy/model/metadata/MetadataRepository.java b/backend/model/metadata/src/main/java/co/airy/model/metadata/MetadataRepository.java new file mode 100644 index 0000000000..f11599dedc --- /dev/null +++ b/backend/model/metadata/src/main/java/co/airy/model/metadata/MetadataRepository.java @@ -0,0 +1,70 @@ +package co.airy.model.metadata; + +import co.airy.avro.communication.Metadata; +import co.airy.uuid.UUIDv5; + +import java.time.Instant; +import java.util.Map; +import java.util.UUID; + +import static co.airy.model.metadata.MetadataKeys.PUBLIC; +import static java.util.stream.Collectors.toMap; + +public class MetadataRepository { + public static Map filterPrefix(Map metadataMap, String prefix) { + return metadataMap + .entrySet() + .stream() + .filter((entry) -> entry.getKey().startsWith(prefix)) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + public static Metadata newConversationMetadata(String conversationId, String key, String value) { + return Metadata.newBuilder() + .setSubject(new Subject("conversation",conversationId).toString()) + .setKey(key) + .setValue(value) + .setTimestamp(Instant.now().toEpochMilli()) + .build(); + } + + public static boolean isConversationMetadata(Metadata metadata) { + return metadata.getSubject().startsWith("conversation:"); + } + + public static Map getConversationInfo(Map metadataMap) { + return filterPrefix(metadataMap, PUBLIC); + } + + public static Metadata newConversationTag(String conversationId, String tagId) { + return Metadata.newBuilder() + .setSubject(new Subject("conversation",conversationId).toString()) + .setKey(String.format("%s.%s", MetadataKeys.TAGS, tagId)) + .setValue("") + .setTimestamp(Instant.now().toEpochMilli()) + .build(); + } + + public static Subject getSubject(Metadata metadata) { + final String subjectString = metadata.getSubject(); + int lastIndexOf = subjectString.lastIndexOf(":"); + + // You do not have to pass an identifier if the namespace you want to + // use metadata for consists of a single object + if (lastIndexOf == -1) { + return new Subject(subjectString, null); + } + + String namespace = subjectString.substring(0, lastIndexOf); + String identifier = subjectString.substring(lastIndexOf + 1); + return new Subject(namespace, identifier); + } + + public static UUID getId(Metadata metadata) { + return UUIDv5.fromNamespaceAndName(metadata.getSubject(), metadata.getKey()); + } + public static UUID getId(Subject subject, String key) { + return UUIDv5.fromNamespaceAndName(subject.toString(), key); + } + +} diff --git a/backend/model/metadata/src/main/java/co/airy/model/metadata/Subject.java b/backend/model/metadata/src/main/java/co/airy/model/metadata/Subject.java new file mode 100644 index 0000000000..aa144f3615 --- /dev/null +++ b/backend/model/metadata/src/main/java/co/airy/model/metadata/Subject.java @@ -0,0 +1,20 @@ +package co.airy.model.metadata; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NonNull; + +@Data +@AllArgsConstructor +public class Subject { + @NonNull + private final String namespace; + private final String identifier; + + public String toString() { + if (identifier == null) { + return namespace; + } + return String.format("%s:%s", namespace, identifier); + } +} diff --git a/backend/sources/chat-plugin/BUILD b/backend/sources/chat-plugin/BUILD index 1558a08e68..9131e2b75f 100644 --- a/backend/sources/chat-plugin/BUILD +++ b/backend/sources/chat-plugin/BUILD @@ -8,10 +8,10 @@ app_deps = [ "//:springboot_actuator", "//:springboot_websocket", "//:springboot_security", - "//backend:channel", - "//backend:message", + "//backend/model/channel:channel", + "//backend/model/message:message", "//lib/java/uuid", - "//lib/java/payload", + "//lib/java/date", "//lib/java/mapping", "//lib/java/spring/web:spring-web", "//lib/java/spring/kafka/core:spring-kafka-core", diff --git a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/ChatController.java b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/ChatController.java index 096c4f35ac..8f19af7188 100644 --- a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/ChatController.java +++ b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/ChatController.java @@ -7,8 +7,10 @@ import co.airy.core.chat_plugin.config.Jwt; import co.airy.core.chat_plugin.payload.AuthenticationRequestPayload; import co.airy.core.chat_plugin.payload.AuthenticationResponsePayload; +import co.airy.core.chat_plugin.payload.ResumeTokenResponsePayload; import co.airy.core.chat_plugin.payload.SendMessageRequestPayload; -import co.airy.payload.response.RequestErrorResponsePayload; +import co.airy.spring.web.payload.EmptyResponsePayload; +import co.airy.spring.web.payload.RequestErrorResponsePayload; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.http.HttpStatus; @@ -17,11 +19,17 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.client.HttpClientErrorException; import javax.validation.Valid; +import java.nio.charset.Charset; import java.time.Instant; +import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; + +import static org.springframework.http.HttpStatus.NOT_FOUND; @RestController public class ChatController { @@ -38,18 +46,47 @@ public ChatController(Stores stores, Jwt jwt, ObjectMapper objectMapper, Mapper } @PostMapping("/chatplugin.authenticate") - ResponseEntity authenticateVisitor(@RequestBody @Valid AuthenticationRequestPayload requestPayload) { - final String channelId = requestPayload.getChannelId().toString(); + ResponseEntity authenticateVisitor(@RequestBody @Valid AuthenticationRequestPayload requestPayload) { + final UUID channelId = requestPayload.getChannelId(); + final String resumeToken = requestPayload.getResumeToken(); + + Principal principal; + List messages = List.of(); + if (resumeToken != null) { + principal = resumeConversation(resumeToken); + messages = stores.getMessages(principal.getConversationId()); + } else if (channelId != null) { + principal = createConversation(channelId.toString()); + } else { + return ResponseEntity.badRequest().body(new EmptyResponsePayload()); + } + + final String authToken = jwt.getAuthToken(principal.getSessionId(), principal.getChannelId()); + + return ResponseEntity.ok(new AuthenticationResponsePayload(authToken, + messages.stream().map(mapper::fromMessage).collect(Collectors.toList()))); + } + + private Principal resumeConversation(String resumeToken) { + return jwt.authenticateResume(resumeToken); + } + + private Principal createConversation(String channelId) { final Channel channel = stores.getChannel(channelId); if (channel == null) { - return ResponseEntity.notFound().build(); + throw new HttpClientErrorException(NOT_FOUND, "Not Found", null, null, Charset.defaultCharset()); } final String sessionId = UUID.randomUUID().toString(); - final String token = jwt.tokenFor(sessionId, channelId); + return new Principal(channelId, sessionId); + } - return ResponseEntity.ok(new AuthenticationResponsePayload(token)); + @PostMapping("/chatplugin.resumeToken") + ResponseEntity getResumeToken(Authentication authentication) { + final Principal principal = (Principal) authentication.getPrincipal(); + final String resumeToken = jwt.getResumeToken(principal.getSessionId(), principal.getChannelId()); + return ResponseEntity.ok(new ResumeTokenResponsePayload(resumeToken)); } @PostMapping("/chatplugin.send") diff --git a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/Mapper.java b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/Mapper.java index 637e7c80fc..23ceeab253 100644 --- a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/Mapper.java +++ b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/Mapper.java @@ -1,12 +1,11 @@ package co.airy.core.chat_plugin; import co.airy.avro.communication.Message; -import co.airy.avro.communication.SenderType; import co.airy.core.chat_plugin.payload.MessageResponsePayload; import co.airy.mapping.ContentMapper; import org.springframework.stereotype.Component; -import static co.airy.payload.format.DateFormat.isoFromMillis; +import static co.airy.date.format.DateFormat.isoFromMillis; @Component public class Mapper { diff --git a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/Stores.java b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/Stores.java index f9925ab1c3..16dec8a576 100644 --- a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/Stores.java +++ b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/Stores.java @@ -5,6 +5,7 @@ import co.airy.avro.communication.DeliveryState; import co.airy.avro.communication.Message; import co.airy.avro.communication.SenderType; +import co.airy.core.chat_plugin.dto.MessagesTreeSet; import co.airy.kafka.schema.application.ApplicationCommunicationChannels; import co.airy.kafka.schema.application.ApplicationCommunicationMessages; import co.airy.kafka.streams.KafkaStreamsWrapper; @@ -24,9 +25,11 @@ import org.springframework.context.ApplicationListener; import org.springframework.stereotype.Component; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.ExecutionException; -import static co.airy.avro.communication.MessageRepository.updateDeliveryState; +import static co.airy.model.message.MessageRepository.updateDeliveryState; @Component public class Stores implements HealthIndicator, ApplicationListener, DisposableBean { @@ -38,6 +41,7 @@ public class Stores implements HealthIndicator, ApplicationListener producer; private final String channelStore = "channel-store"; + private final String messagesStore = "messages-store"; Stores(KafkaStreamsWrapper streams, KafkaProducer producer, @@ -53,6 +57,14 @@ private void startStream() { final KStream messageStream = builder.stream(applicationCommunicationMessages) .filter((messageId, message) -> "chat_plugin".equals(message.getSource())); + // Messages store + messageStream + .groupBy((messageId, message) -> message.getConversationId()) + .aggregate(MessagesTreeSet::new, ((key, value, aggregate) -> { + aggregate.add(value); + return aggregate; + }), Materialized.as(messagesStore)); + // Client Echoes messageStream.filter((messageId, message) -> message.getSenderType().equals(SenderType.SOURCE_CONTACT)) .peek((messageId, message) -> webSocketController.onNewMessage(message)); @@ -89,11 +101,23 @@ private ReadOnlyKeyValueStore getChannelsStore() { return streams.acquireLocalStore(channelStore); } + public ReadOnlyKeyValueStore getMessagesStore() { + return streams.acquireLocalStore(messagesStore); + } + public Channel getChannel(String channelId) { final ReadOnlyKeyValueStore store = getChannelsStore(); return store.get(channelId); } + public List getMessages(String conversationId) { + final ReadOnlyKeyValueStore messagesStore = getMessagesStore(); + + final MessagesTreeSet messagesTreeSet = messagesStore.get(conversationId); + + return messagesTreeSet == null ? List.of() : new ArrayList<>(messagesTreeSet); + } + @Override public void destroy() { if (streams != null) { diff --git a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/config/Jwt.java b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/config/Jwt.java index e8fee57a4f..a54b360c82 100644 --- a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/config/Jwt.java +++ b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/config/Jwt.java @@ -28,13 +28,15 @@ public class Jwt { private static final Logger log = AiryLoggerFactory.getLogger(Jwt.class); public static final String SESSION_ID_CLAIM = "session_id"; public static final String CHANNEL_ID_CLAIM = "channel_id"; + public static final String RESUME_SESSION_ID_CLAIM = "resume_session_id"; + public static final String RESUME_CHANNEL_ID_CLAIM = "resume_channel_id"; private final Key signingKey; public Jwt(@Value("${chat-plugin.auth.jwt-secret}") String tokenKey) { this.signingKey = parseSigningKey(tokenKey); } - public String tokenFor(String sessionId, String channelId) { + public String getAuthToken(String sessionId, String channelId) { Date now = Date.from(Instant.now()); Map claims = new HashMap<>(); @@ -54,14 +56,32 @@ public String tokenFor(String sessionId, String channelId) { return builder.compact(); } + public String getResumeToken(String sessionId, String channelId) { + Date now = Date.from(Instant.now()); + + Map claims = new HashMap<>(); + claims.put(RESUME_SESSION_ID_CLAIM, sessionId); + claims.put(RESUME_CHANNEL_ID_CLAIM, channelId); + + JwtBuilder builder = Jwts.builder() + .setId(sessionId) + .setSubject(sessionId) + .setIssuedAt(now) + .addClaims(claims) + .signWith(signingKey, SignatureAlgorithm.HS256); + + Date exp = Date.from(Instant.now().plus(Duration.ofDays(7))); + builder.setExpiration(exp); + + return builder.compact(); + } + public Principal authenticate(final String authHeader) throws HttpClientErrorException.Unauthorized { Claims claims = null; - if (authHeader != null) { - try { - claims = extractClaims(authHeader); - } catch (Exception e) { - log.error("Failed to extract claims from token: " + e.getMessage()); - } + try { + claims = extractClaims(authHeader); + } catch (Exception e) { + log.error("Failed to extract claims from token: " + e.getMessage()); } if (claims == null) { @@ -78,6 +98,28 @@ public Principal authenticate(final String authHeader) throws HttpClientErrorExc } } + public Principal authenticateResume(final String resumeToken) throws HttpClientErrorException.Unauthorized { + Claims claims = null; + try { + claims = extractClaims(resumeToken); + } catch (Exception e) { + log.error("Failed to extract claims from token: " + e.getMessage()); + } + + if (claims == null) { + throw new HttpClientErrorException(UNAUTHORIZED, "Unauthorized", null, null, Charset.defaultCharset()); + } + + try { + final String sessionId = (String) claims.get(RESUME_SESSION_ID_CLAIM); + final String channelId = (String) claims.get(RESUME_CHANNEL_ID_CLAIM); + return new Principal(channelId, sessionId); + } catch (Exception e) { + log.error(e.getMessage()); + throw new HttpClientErrorException(UNAUTHORIZED, "Unauthorized", null, null, Charset.defaultCharset()); + } + } + private Key parseSigningKey(String tokenKey) { byte[] apiKeySecretBytes = DatatypeConverter.parseBase64Binary(tokenKey); return new SecretKeySpec(apiKeySecretBytes, SignatureAlgorithm.HS256.getJcaName()); diff --git a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/dto/MessagesTreeSet.java b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/dto/MessagesTreeSet.java new file mode 100644 index 0000000000..22f7e289c3 --- /dev/null +++ b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/dto/MessagesTreeSet.java @@ -0,0 +1,14 @@ +package co.airy.core.chat_plugin.dto; + +import co.airy.avro.communication.Message; +import com.fasterxml.jackson.annotation.JsonCreator; + +import java.util.Comparator; +import java.util.TreeSet; + +public class MessagesTreeSet extends TreeSet { + @JsonCreator + public MessagesTreeSet() { + super(Comparator.comparing(Message::getSentAt)); + } +} diff --git a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/AuthenticationRequestPayload.java b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/AuthenticationRequestPayload.java index b91b42e414..da19d81987 100644 --- a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/AuthenticationRequestPayload.java +++ b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/AuthenticationRequestPayload.java @@ -3,13 +3,11 @@ import lombok.Data; import lombok.NoArgsConstructor; -import javax.validation.constraints.NotNull; import java.util.UUID; @Data @NoArgsConstructor public class AuthenticationRequestPayload { - @NotNull private UUID channelId; + private String resumeToken; } - diff --git a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/AuthenticationResponsePayload.java b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/AuthenticationResponsePayload.java index cfa5edfb31..a4f0a7feaa 100644 --- a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/AuthenticationResponsePayload.java +++ b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/AuthenticationResponsePayload.java @@ -4,10 +4,12 @@ import lombok.Data; import lombok.NoArgsConstructor; +import java.util.List; + @Data @NoArgsConstructor @AllArgsConstructor public class AuthenticationResponsePayload { private String token; + private List messages; } - diff --git a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/MessageResponsePayload.java b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/MessageResponsePayload.java index a7287e8f72..150e27f2e4 100644 --- a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/MessageResponsePayload.java +++ b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/MessageResponsePayload.java @@ -7,6 +7,7 @@ import lombok.NoArgsConstructor; import java.io.Serializable; +import java.util.List; @Data @Builder @@ -14,7 +15,7 @@ @AllArgsConstructor public class MessageResponsePayload implements Serializable { private String id; - private Content content; + private List content; private String state; private String senderType; private String sentAt; diff --git a/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/ResumeTokenResponsePayload.java b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/ResumeTokenResponsePayload.java new file mode 100644 index 0000000000..25f7469e6c --- /dev/null +++ b/backend/sources/chat-plugin/src/main/java/co/airy/core/chat_plugin/payload/ResumeTokenResponsePayload.java @@ -0,0 +1,12 @@ +package co.airy.core.chat_plugin.payload; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ResumeTokenResponsePayload { + private String resumeToken; +} diff --git a/backend/sources/chat-plugin/src/main/resources/application.properties b/backend/sources/chat-plugin/src/main/resources/application.properties index 9699cca357..a51befc8ca 100644 --- a/backend/sources/chat-plugin/src/main/resources/application.properties +++ b/backend/sources/chat-plugin/src/main/resources/application.properties @@ -1,3 +1,4 @@ kafka.brokers=${KAFKA_BROKERS} kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} +kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS} chat-plugin.auth.jwt-secret=${JWT_SECRET} diff --git a/backend/sources/chat-plugin/src/test/java/co/airy/core/chat_plugin/ChatControllerTest.java b/backend/sources/chat-plugin/src/test/java/co/airy/core/chat_plugin/ChatControllerTest.java index 6a31b8214e..0f454858c6 100644 --- a/backend/sources/chat-plugin/src/test/java/co/airy/core/chat_plugin/ChatControllerTest.java +++ b/backend/sources/chat-plugin/src/test/java/co/airy/core/chat_plugin/ChatControllerTest.java @@ -44,6 +44,8 @@ import static co.airy.core.chat_plugin.WebSocketController.QUEUE_MESSAGE; import static co.airy.test.Timing.retryOnException; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -71,9 +73,8 @@ public class ChatControllerTest { private static KafkaTestHelper kafkaTestHelper; private static final ApplicationCommunicationMessages applicationCommunicationMessages = new ApplicationCommunicationMessages(); private static final ApplicationCommunicationChannels applicationCommunicationChannels = new ApplicationCommunicationChannels(); - private static boolean testDataInitialized = false; - private final Channel channel = Channel.newBuilder() + private static final Channel channel = Channel.newBuilder() .setConnectionState(ChannelConnectionState.CONNECTED) .setId(UUID.randomUUID().toString()) .setName("Chat Plugin") @@ -89,6 +90,8 @@ static void beforeAll() throws Exception { ); kafkaTestHelper.beforeAll(); + + kafkaTestHelper.produceRecord(new ProducerRecord<>(applicationCommunicationChannels.name(), channel.getId(), channel)); } @AfterAll @@ -98,12 +101,6 @@ static void afterAll() throws Exception { @BeforeEach void beforeEach() throws Exception { - if (testDataInitialized) { - return; - } - testDataInitialized = true; - kafkaTestHelper.produceRecord(new ProducerRecord<>(applicationCommunicationChannels.name(), channel.getId(), channel)); - retryOnException(() -> mvc.perform(get("/actuator/health")).andExpect(status().isOk()), "Application is not healthy"); } @@ -125,19 +122,61 @@ void authenticateSendAndReceive() throws Exception { final String messageText = "answer is 42"; String sendMessagePayload = "{\"message\": { \"text\": \"" + messageText + "\" }}"; - retryOnException(() -> - mvc.perform(post("/chatplugin.send") + retryOnException(() -> mvc.perform(post("/chatplugin.send") .headers(buildHeaders(token)) .content(sendMessagePayload)) .andExpect(status().isOk()) - .andExpect(jsonPath("$.content.text", containsString(messageText))), - "Message was not sent" - ); + .andExpect(jsonPath("$.content[0].text", containsString(messageText))), + "Message was not sent"); final MessageUpsertPayload messageUpsertPayload = messageFuture.get(); assertNotNull(messageUpsertPayload); - assertThat(((Text) messageUpsertPayload.getMessage().getContent()).getText(), containsString(messageText)); + final Text text = (Text) messageUpsertPayload.getMessage().getContent().get(0); + assertThat(text.getText(), containsString(messageText)); + } + + @Test + void canResumeConversation() throws Exception { + final String authPayload = "{\"channel_id\":\"" + channel.getId() + "\"}"; + + String response = mvc.perform(post("/chatplugin.authenticate") + .header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) + .content(authPayload)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.token", is(not(nullValue())))) + .andReturn().getResponse().getContentAsString(); + JsonNode jsonNode = new ObjectMapper().readTree(response); + final String authToken = jsonNode.get("token").textValue(); + + final String messageText = "Talk to you later!"; + final String sendMessagePayload = "{\"message\": { \"text\": \"" + messageText + "\" }}"; + mvc.perform(post("/chatplugin.send") + .headers(buildHeaders(authToken)) + .content(sendMessagePayload)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.content[0].text", containsString(messageText))); + + response = mvc.perform(post("/chatplugin.resumeToken") + .headers(buildHeaders(authToken)) + .content("{}")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.resume_token", is(not(nullValue())))) + .andReturn().getResponse().getContentAsString(); + jsonNode = new ObjectMapper().readTree(response); + final String resumeToken = jsonNode.get("resume_token").textValue(); + + + retryOnException(() -> { + final String resumePayload = "{\"resume_token\":\"" + resumeToken + "\"}"; + mvc.perform(post("/chatplugin.authenticate") + .header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) + .content(resumePayload)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.token", is(not(nullValue())))) + .andExpect(jsonPath("$.messages", hasSize(1))) + .andExpect(jsonPath("$.messages[0].content[0].text", equalTo(messageText))); + }, "Did not resume conversation"); } private HttpHeaders buildHeaders(String jwtToken) { @@ -160,8 +199,7 @@ public StompSession connect(String jwtToken, int port) throws ExecutionException WebSocketHttpHeaders httpHeaders = new WebSocketHttpHeaders(); - return stompClient.connect("ws://localhost:" + port + "/ws.chatplugin", httpHeaders, connectHeaders, new StompSessionHandlerAdapter() { - }).get(); + return stompClient.connect("ws://localhost:" + port + "/ws.chatplugin", httpHeaders, connectHeaders, new StompSessionHandlerAdapter() {}).get(); } public CompletableFuture subscribe(String jwtToken, int port, Class payloadType, String topic) throws ExecutionException, InterruptedException { diff --git a/backend/sources/facebook/sender/BUILD b/backend/sources/facebook/connector/BUILD similarity index 71% rename from backend/sources/facebook/sender/BUILD rename to backend/sources/facebook/connector/BUILD index 08ddd19d77..53cf96932f 100644 --- a/backend/sources/facebook/sender/BUILD +++ b/backend/sources/facebook/connector/BUILD @@ -4,15 +4,20 @@ load("//tools/build:container_push.bzl", "container_push") app_deps = [ "//backend:base_app", - "//backend:channel", - "//backend:message", + "//backend/model/channel:channel", + "//backend/model/message:message", + "//backend/model/metadata:metadata", + "//lib/java/uuid", "//lib/java/log", + "//lib/java/spring/web:spring-web", + "//lib/java/spring/auth:spring-auth", "//lib/java/spring/kafka/core:spring-kafka-core", "//lib/java/spring/kafka/streams:spring-kafka-streams", + "//lib/java/kafka/schema:source-facebook-events", ] springboot( - name = "sender", + name = "connector", srcs = glob(["src/main/java/**/*.java"]), main_class = "co.airy.spring.core.AirySpringBootApplication", deps = app_deps, @@ -34,5 +39,5 @@ springboot( container_push( registry = "ghcr.io/airyhq/sources", - repository = "facebook-sender", + repository = "facebook-connector", ) diff --git a/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/ChannelsController.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/ChannelsController.java new file mode 100644 index 0000000000..07f505d7c9 --- /dev/null +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/ChannelsController.java @@ -0,0 +1,107 @@ +package co.airy.core.sources.facebook; + +import co.airy.avro.communication.Channel; +import co.airy.avro.communication.ChannelConnectionState; +import co.airy.core.sources.facebook.payload.PageInfoResponsePayload; +import co.airy.core.sources.facebook.payload.ConnectRequestPayload; +import co.airy.core.sources.facebook.payload.ExploreRequestPayload; +import co.airy.core.sources.facebook.payload.ExploreResponsePayload; +import co.airy.core.sources.facebook.api.Api; +import co.airy.core.sources.facebook.api.ApiException; +import co.airy.core.sources.facebook.api.model.PageWithConnectInfo; +import co.airy.spring.web.payload.RequestErrorResponsePayload; +import co.airy.uuid.UUIDv5; +import org.apache.kafka.streams.state.KeyValueIterator; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import javax.validation.Valid; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static co.airy.model.channel.ChannelPayload.fromChannel; +import static java.util.stream.Collectors.toList; + +@RestController +public class ChannelsController { + + private final Api api; + private final Stores stores; + + public ChannelsController(Api api, Stores stores) { + this.api = api; + this.stores = stores; + } + + @PostMapping("/facebook.explore") + ResponseEntity explore(@RequestBody @Valid ExploreRequestPayload requestPayload) { + try { + final List pagesInfo = api.getPagesInfo(requestPayload.getAuthToken()); + + final KeyValueIterator iterator = stores.getChannelsStore().all(); + + List channels = new ArrayList<>(); + iterator.forEachRemaining(kv -> channels.add(kv.value)); + + final List connectedSourceIds = channels + .stream() + .filter((channel -> ChannelConnectionState.CONNECTED.equals(channel.getConnectionState()))) + .map(Channel::getSourceChannelId) + .collect(toList()); + + return ResponseEntity.ok( + new ExploreResponsePayload( + pagesInfo.stream() + .map((page) -> PageInfoResponsePayload.builder() + .pageId(page.getId()) + .name(page.getNameWithLocationDescriptor()) + .imageUrl(page.getPicture().getData().getUrl()) + .connected(connectedSourceIds.contains(page.getId())) + .build() + ).collect(toList()) + ) + ); + } catch (ApiException e) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new RequestErrorResponsePayload(e.getMessage())); + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build(); + } + } + + @PostMapping("/facebook.connect") + ResponseEntity connect(@RequestBody @Valid ConnectRequestPayload requestPayload) { + final String token = requestPayload.getPageToken(); + final String pageId = requestPayload.getPageId(); + + final String channelId = UUIDv5.fromNamespaceAndName("facebook", pageId).toString(); + + try { + final String longLivingUserToken = api.exchangeToLongLivingUserAccessToken(token); + final PageWithConnectInfo fbPageWithConnectInfo = api.getPageForUser(pageId, longLivingUserToken); + + api.connectPageToApp(fbPageWithConnectInfo.getAccessToken()); + + final Channel channel = Channel.newBuilder() + .setId(channelId) + .setConnectionState(ChannelConnectionState.CONNECTED) + .setImageUrl(Optional.ofNullable(requestPayload.getImageUrl()).orElse(fbPageWithConnectInfo.getPicture().getData().getUrl())) + .setName(Optional.ofNullable(requestPayload.getName()).orElse(fbPageWithConnectInfo.getNameWithLocationDescriptor())) + .setSource("facebook") + .setSourceChannelId(pageId) + .setToken(token) + .build(); + + stores.storeChannel(channel); + + return ResponseEntity.ok(fromChannel(channel)); + } catch (ApiException e) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new RequestErrorResponsePayload(e.getMessage())); + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build(); + } + } +} diff --git a/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/Connector.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/Connector.java new file mode 100644 index 0000000000..8d5c2c561a --- /dev/null +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/Connector.java @@ -0,0 +1,130 @@ +package co.airy.core.sources.facebook; + +import co.airy.avro.communication.DeliveryState; +import co.airy.avro.communication.Message; +import co.airy.avro.communication.Metadata; +import co.airy.model.metadata.MetadataKeys; +import co.airy.core.sources.facebook.dto.SendMessageRequest; +import co.airy.core.sources.facebook.dto.Conversation; +import co.airy.core.sources.facebook.api.model.SendMessagePayload; +import co.airy.core.sources.facebook.api.model.UserProfile; +import co.airy.core.sources.facebook.api.Api; +import co.airy.core.sources.facebook.api.ApiException; +import co.airy.core.sources.facebook.api.Mapper; +import co.airy.log.AiryLoggerFactory; +import co.airy.spring.auth.IgnoreAuthPattern; +import co.airy.spring.web.filters.RequestLoggingIgnorePatterns; +import org.apache.kafka.streams.KeyValue; +import org.slf4j.Logger; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static co.airy.model.message.MessageRepository.updateDeliveryState; +import static co.airy.model.metadata.MetadataKeys.Source.ContactFetchState.failed; +import static co.airy.model.metadata.MetadataKeys.Source.ContactFetchState.ok; +import static co.airy.model.metadata.MetadataRepository.getId; +import static co.airy.model.metadata.MetadataRepository.newConversationMetadata; + +@Component +public class Connector { + private static final Logger log = AiryLoggerFactory.getLogger(Connector.class); + + private final Api api; + private final Mapper mapper; + + Connector(Api api, Mapper mapper) { + this.api = api; + this.mapper = mapper; + } + + public Message sendMessage(SendMessageRequest sendMessageRequest) { + final Message message = sendMessageRequest.getMessage(); + final Conversation conversation = sendMessageRequest.getConversation(); + + try { + final String pageToken = conversation.getChannel().getToken(); + final SendMessagePayload fbSendMessagePayload = mapper.fromSendMessageRequest(sendMessageRequest); + + api.sendMessage(pageToken, fbSendMessagePayload); + + updateDeliveryState(message, DeliveryState.DELIVERED); + return message; + } catch (ApiException e) { + log.error(String.format("Failed to send a message to Facebook \n SendMessageRequest: %s \n Error Message: %s \n", sendMessageRequest, e.getMessage()), e); + } catch (Exception e) { + log.error(String.format("Failed to send a message to Facebook \n SendMessageRequest: %s", sendMessageRequest), e); + } + + updateDeliveryState(message, DeliveryState.FAILED); + return message; + } + + public boolean needsMetadataFetched(Conversation conversation) { + final Map metadata = conversation.getMetadata(); + final String fetchState = metadata.get(MetadataKeys.Source.Contact.FETCH_STATE); + + return !ok.toString().equals(fetchState) && !failed.toString().equals(fetchState); + } + + public List> fetchMetadata(String conversationId, Conversation conversation) { + final UserProfile profile = getProfile(conversation); + + final List> recordList = new ArrayList<>(); + + if (profile.getFirstName() != null) { + final Metadata firstName = newConversationMetadata(conversationId, MetadataKeys.Source.Contact.FIRST_NAME, profile.getFirstName()); + recordList.add(KeyValue.pair(getId(firstName).toString(), firstName)); + } + + if (profile.getLastName() != null) { + final Metadata lastName = newConversationMetadata(conversationId, MetadataKeys.Source.Contact.LAST_NAME, profile.getLastName()); + recordList.add(KeyValue.pair(getId(lastName).toString(), lastName)); + } + + if (profile.getProfilePic() != null) { + final Metadata avatarUrl = newConversationMetadata(conversationId, MetadataKeys.Source.Contact.AVATAR_URL, profile.getProfilePic()); + recordList.add(KeyValue.pair(getId(avatarUrl).toString(), avatarUrl)); + } + + final String newFetchState = recordList.size() > 0 ? ok.toString() : failed.toString(); + final String oldFetchState = conversation.getMetadata().get(MetadataKeys.Source.Contact.FETCH_STATE); + + // Only update fetch state if there has been a change + if (!newFetchState.equals(oldFetchState)) { + final Metadata fetchState = newConversationMetadata(conversationId, MetadataKeys.Source.Contact.FETCH_STATE, newFetchState); + recordList.add(KeyValue.pair(getId(fetchState).toString(), fetchState)); + } + + return recordList; + } + + public UserProfile getProfile(Conversation conversation) { + final String sourceConversationId = conversation.getSourceConversationId(); + final String token = conversation.getChannel().getToken(); + try { + return api.getProfileFromContact(sourceConversationId, token); + } catch (Exception profileApiException) { + log.error("Profile api failed", profileApiException); + try { + return api.getProfileFromParticipants(sourceConversationId, token); + } catch (Exception participantApiException) { + log.error("Participant api failed", participantApiException); + return new UserProfile(); + } + } + } + + @Bean + public IgnoreAuthPattern ignoreAuthPattern() { + return new IgnoreAuthPattern("/facebook"); + } + + @Bean + public RequestLoggingIgnorePatterns requestLoggingIgnorePatterns() { + return new RequestLoggingIgnorePatterns(List.of("/facebook")); + } +} diff --git a/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/Stores.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/Stores.java new file mode 100644 index 0000000000..5aaf241f4e --- /dev/null +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/Stores.java @@ -0,0 +1,146 @@ +package co.airy.core.sources.facebook; + +import co.airy.avro.communication.Channel; +import co.airy.avro.communication.ChannelConnectionState; +import co.airy.avro.communication.DeliveryState; +import co.airy.avro.communication.Message; +import co.airy.avro.communication.Metadata; +import co.airy.avro.communication.SenderType; +import co.airy.core.sources.facebook.dto.SendMessageRequest; +import co.airy.core.sources.facebook.dto.Conversation; +import co.airy.kafka.schema.application.ApplicationCommunicationChannels; +import co.airy.kafka.schema.application.ApplicationCommunicationMessages; +import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; +import co.airy.kafka.streams.KafkaStreamsWrapper; +import org.apache.avro.specific.SpecificRecordBase; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Suppressed; +import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; + +import static co.airy.model.metadata.MetadataRepository.getSubject; +import static co.airy.model.metadata.MetadataRepository.isConversationMetadata; + +@Service +public class Stores implements ApplicationListener, DisposableBean { + private static final String appId = "sources.facebook.ConnectorStores"; + + private final KafkaStreamsWrapper streams; + private final String channelsStore = "channels-store"; + private final String applicationCommunicationChannels = new ApplicationCommunicationChannels().name(); + private final KafkaProducer producer; + private final Connector connector; + + public Stores(KafkaStreamsWrapper streams, KafkaProducer producer, Connector connector) { + this.streams = streams; + this.producer = producer; + this.connector = connector; + } + + @Override + public void onApplicationEvent(ApplicationStartedEvent applicationStartedEvent) { + final StreamsBuilder builder = new StreamsBuilder(); + + KStream channelStream = builder.stream(applicationCommunicationChannels); + + channelStream.toTable(Materialized.as(channelsStore)); + + // Channels table + KTable channelsTable = channelStream + .filter((sourceChannelId, channel) -> "facebook".equalsIgnoreCase(channel.getSource()) + && channel.getConnectionState().equals(ChannelConnectionState.CONNECTED)).toTable(); + + // Facebook messaging stream by conversation-id + final KStream messageStream = builder.stream(new ApplicationCommunicationMessages().name()) + .filter((messageId, message) -> "facebook".equalsIgnoreCase(message.getSource())) + .selectKey((messageId, message) -> message.getConversationId()); + + // Metadata table + final KTable> metadataTable = builder.table(new ApplicationCommunicationMetadata().name()) + .filter((metadataId, metadata) -> isConversationMetadata(metadata)) + .groupBy((metadataId, metadata) -> KeyValue.pair(getSubject(metadata).getIdentifier(), metadata)) + .aggregate(HashMap::new, (conversationId, metadata, aggregate) -> { + aggregate.put(metadata.getKey(), metadata.getValue()); + return aggregate; + }, (conversationId, metadata, aggregate) -> { + aggregate.remove(metadata.getKey()); + return aggregate; + }); + + // Conversation table + final KTable conversationTable = messageStream + .groupByKey() + .aggregate(Conversation::new, + (conversationId, message, aggregate) -> { + if (SenderType.SOURCE_CONTACT.equals(message.getSenderType())) { + aggregate.setSourceConversationId(message.getSenderId()); + } + + aggregate.setChannelId(message.getChannelId()); + + return aggregate; + }) + .join(channelsTable, Conversation::getChannelId, (aggregate, channel) -> { + aggregate.setChannel(channel); + return aggregate; + }); + + // Send outbound messages + messageStream.filter((messageId, message) -> DeliveryState.PENDING.equals(message.getDeliveryState())) + .join(conversationTable, (message, conversation) -> new SendMessageRequest(conversation, message)) + .mapValues(connector::sendMessage) + .to(new ApplicationCommunicationMessages().name()); + + // Fetch missing metadata + conversationTable + // To avoid any redundant fetch contact operations the suppression interval should + // be higher than the timeout of the Facebook API + .suppress(Suppressed.untilTimeLimit(Duration.ofMillis(streams.getSuppressIntervalInMs()), Suppressed.BufferConfig.unbounded())) + .toStream() + .leftJoin(metadataTable, (conversation, metadataMap) -> conversation + .toBuilder() + .metadata(new HashMap<>(Optional.ofNullable(metadataMap).orElse(Map.of()))) + .build()) + .filter((k, v) -> connector.needsMetadataFetched(v)) + .flatMap(connector::fetchMetadata) + .to(new ApplicationCommunicationMetadata().name()); + + streams.start(builder.build(), appId); + } + + public ReadOnlyKeyValueStore getChannelsStore() { + return streams.acquireLocalStore(channelsStore); + } + + public void storeChannel(Channel channel) throws ExecutionException, InterruptedException { + producer.send(new ProducerRecord<>(applicationCommunicationChannels, channel.getId(), channel)).get(); + } + + @Override + public void destroy() { + if (streams != null) { + streams.close(); + } + } + + // visible for testing + KafkaStreams.State getStreamState() { + return streams.state(); + } +} diff --git a/backend/sources/facebook/webhook/src/main/java/co/airy/core/sources/facebook/FacebookWebhook.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/WebhookController.java similarity index 75% rename from backend/sources/facebook/webhook/src/main/java/co/airy/core/sources/facebook/FacebookWebhook.java rename to backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/WebhookController.java index bd1527cbfa..8923aa26ad 100644 --- a/backend/sources/facebook/webhook/src/main/java/co/airy/core/sources/facebook/FacebookWebhook.java +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/WebhookController.java @@ -1,7 +1,6 @@ package co.airy.core.sources.facebook; import co.airy.kafka.schema.source.SourceFacebookEvents; -import co.airy.spring.kafka.healthcheck.ProducerHealthCheck; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; @@ -9,9 +8,6 @@ import org.apache.kafka.common.serialization.StringSerializer; import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.actuate.health.Health; -import org.springframework.boot.actuate.health.HealthIndicator; -import org.springframework.boot.actuate.health.Status; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; @@ -25,17 +21,14 @@ import java.util.UUID; @RestController -public class FacebookWebhook implements HealthIndicator, DisposableBean { +public class WebhookController implements DisposableBean { private final String sourceFacebookEvents = new SourceFacebookEvents().name(); private final String webhookSecret; private final Producer producer; - private final ProducerHealthCheck producerHealthCheck; - FacebookWebhook(ProducerHealthCheck producerHealthCheck, - @Value("${kafka.brokers}") String brokers, - @Value("${facebook.webhook-secret}") String webhookSecret) { - this.producerHealthCheck = producerHealthCheck; + WebhookController(@Value("${kafka.brokers}") String brokers, + @Value("${facebook.webhook-secret}") String webhookSecret) { this.webhookSecret = webhookSecret; final Properties props = new Properties(); @@ -48,15 +41,6 @@ public class FacebookWebhook implements HealthIndicator, DisposableBean { producer = new KafkaProducer<>(props); } - public Health health() { - try { - producerHealthCheck.sendHealthCheck(); - return Health.status(Status.UP).build(); - } catch (Exception e) { - return Health.down(e).build(); - } - } - // https://developers.facebook.com/docs/graph-api/webhooks/getting-started @GetMapping("/facebook") int verify(@RequestParam(value = "hub.challenge") int challenge, @RequestParam(value = "hub.verify_token") String verifyToken) { diff --git a/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/Api.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/Api.java new file mode 100644 index 0000000000..2112768cbe --- /dev/null +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/Api.java @@ -0,0 +1,192 @@ +package co.airy.core.sources.facebook.api; + +import co.airy.core.sources.facebook.api.model.LongLivingUserAccessToken; +import co.airy.core.sources.facebook.api.model.PageWithConnectInfo; +import co.airy.core.sources.facebook.api.model.Pages; +import co.airy.core.sources.facebook.api.model.Participants; +import co.airy.core.sources.facebook.api.model.SendMessagePayload; +import co.airy.core.sources.facebook.api.model.UserProfile; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.context.ApplicationListener; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.ClientHttpResponse; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.stereotype.Service; +import org.springframework.web.client.ResponseErrorHandler; +import org.springframework.web.client.RestTemplate; + +import java.io.IOException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/* + @see https://developers.facebook.com/docs/messenger-platform/reference/send-api/ + */ +@Service +public class Api implements ApplicationListener { + private final RestTemplateBuilder restTemplateBuilder; + private final ObjectMapper objectMapper; + + private RestTemplate restTemplate; + + private static final String subscribedFields = "messages,messaging_postbacks,messaging_optins,message_deliveries,message_reads,messaging_payments,messaging_pre_checkouts,messaging_checkout_updates,messaging_account_linking,messaging_referrals,message_echoes,messaging_game_plays,standby,messaging_handovers,messaging_policy_enforcement,message_reactions,inbox_labels"; + private static final String baseUrl = "https://graph.facebook.com/v3.2"; + private static final String requestTemplate = baseUrl + "/me/messages?access_token=%s"; + private final String pageFields = "fields=id,name_with_location_descriptor,access_token,picture,is_webhooks_subscribed"; + + private final HttpHeaders httpHeaders = new HttpHeaders(); + private final String appId; + private final String apiSecret; + private static final String errorMessageTemplate = + "Exception while sending a message to Facebook: \n" + + "Http Status Code: %s \n" + + "Error Message: %s \n"; + + public Api(ObjectMapper objectMapper, RestTemplateBuilder restTemplateBuilder, + @Value("${facebook.app-id}") String appId, + @Value("${facebook.app-secret}") String apiSecret + ) { + httpHeaders.setContentType(MediaType.APPLICATION_JSON); + this.objectMapper = objectMapper; + this.restTemplateBuilder = restTemplateBuilder; + this.appId = appId; + this.apiSecret = apiSecret; + } + + public void sendMessage(final String pageToken, SendMessagePayload sendMessagePayload) { + String fbReqUrl = String.format(requestTemplate, pageToken); + + restTemplate.postForEntity(fbReqUrl, new HttpEntity<>(sendMessagePayload, httpHeaders), FbSendMessageResponse.class); + } + + public List getPagesInfo(String accessToken) throws Exception { + String pagesUrl = String.format(baseUrl + "/me/accounts?%s&access_token=%s", pageFields, accessToken); + + boolean hasMorePages = true; + List pageList = new ArrayList<>(); + while (hasMorePages) { + Pages fbPages = apiResponse(pagesUrl, HttpMethod.GET, Pages.class); + if (fbPages.getPaging() != null && fbPages.getPaging().getNext() != null) { + pagesUrl = URLDecoder.decode(fbPages.getPaging().getNext(), StandardCharsets.UTF_8); + } else { + hasMorePages = false; + } + pageList.addAll(fbPages.getData()); + } + return pageList; + } + + private T apiResponse(String url, HttpMethod method, Class clazz) throws Exception { + ResponseEntity responseEntity = restTemplate.exchange(url, method, null, String.class); + + return objectMapper.readValue(responseEntity.getBody(), clazz); + } + + + // See "Retrieving a Person's Profile" in https://developers.facebook.com/docs/messenger-platform/identity/user-profile + public UserProfile getProfileFromContact(String sourceConversationId, String token) { + String reqUrl = String.format(baseUrl + "/%s?fields=first_name,last_name,profile_pic&access_token=%s", + sourceConversationId, token); + ResponseEntity responseEntity = restTemplate.getForEntity(reqUrl, UserProfile.class); + if (responseEntity.getStatusCode() != HttpStatus.OK) { + throw new ApiException("Call unsuccessful, received HTTP status " + responseEntity.getStatusCodeValue()); + } + return responseEntity.getBody(); + } + + // See https://developers.facebook.com/docs/graph-api/reference/v9.0/conversation#edges + public UserProfile getProfileFromParticipants(String sourceConversationId, String token) { + String reqUrl = String.format("https://graph.facebook.com/v9.0/me/conversations?user_id=%s&fields=participants&access_token=%s", + sourceConversationId, token); + + ResponseEntity responseEntity = restTemplate.getForEntity(reqUrl, Participants.class); + if (responseEntity.getBody() == null || responseEntity.getStatusCode() != HttpStatus.OK) { + throw new ApiException("Call unsuccessful"); + } + + return fromParticipants(responseEntity.getBody(), sourceConversationId); + } + + public PageWithConnectInfo getPageForUser(final String pageId, final String accessToken) throws Exception { + final String pageUrl = String.format(baseUrl + "/%s?%s&access_token=%s", pageId, pageFields, accessToken); + + return apiResponse(pageUrl, HttpMethod.GET, PageWithConnectInfo.class); + } + + public void connectPageToApp(String pageToken) throws Exception { + String apiUrl = String.format(baseUrl + "/me/subscribed_apps?access_token=%s&subscribed_fields=%s", pageToken, subscribedFields); + apiResponse(apiUrl, HttpMethod.POST, Map.class); + } + + public String exchangeToLongLivingUserAccessToken(String userAccessToken) throws Exception { + String apiUrl = String.format(baseUrl + "/oauth/access_token?grant_type=fb_exchange_token&client_id=%s&client_secret=%s&fb_exchange_token=%s", appId, apiSecret, userAccessToken); + return apiResponse(apiUrl, HttpMethod.GET, LongLivingUserAccessToken.class).getAccessToken(); + } + + private UserProfile fromParticipants(Participants participants, String psId) { + return participants.getData() + .get(0) + .getParticipants() + .getData() + .stream() + .filter((participantEntry -> psId.equals(participantEntry.getId()))) + .map((userParticipant) -> { + final String name = userParticipant.getName(); + final List splits = new ArrayList<>(Arrays.asList(name.split(" "))); + + final String firstName = splits.get(0); + splits.remove(0); + String lastName = String.join(" ", splits); + + return UserProfile.builder().firstName(firstName).lastName(lastName).build(); + }) + .findFirst() + .get(); + } + + + @Override + public void onApplicationEvent(ApplicationReadyEvent event) { + restTemplate = restTemplateBuilder + .errorHandler(new ResponseErrorHandler() { + @Override + public boolean hasError(ClientHttpResponse response) throws IOException { + return response.getRawStatusCode() != HttpStatus.OK.value(); + } + + @Override + public void handleError(ClientHttpResponse response) throws IOException { + throw new ApiException(String.format(errorMessageTemplate, response.getRawStatusCode(), new String(response.getBody().readAllBytes()))); + } + }) + .additionalMessageConverters(new MappingJackson2HttpMessageConverter(objectMapper)) + .build(); + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + private static class FbSendMessageResponse { + @JsonProperty("recipient_id") + private String recipientId; + + @JsonProperty("message_id") + private String messageId; + } +} diff --git a/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/ApiException.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/ApiException.java similarity index 56% rename from backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/ApiException.java rename to backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/ApiException.java index 862be54013..69dbceff1a 100644 --- a/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/ApiException.java +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/ApiException.java @@ -1,10 +1,6 @@ -package co.airy.core.sources.facebook; +package co.airy.core.sources.facebook.api; public class ApiException extends RuntimeException { - public ApiException() { - super(); - } - public ApiException(String msg) { super(msg); } diff --git a/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/services/Mapper.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/Mapper.java similarity index 80% rename from backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/services/Mapper.java rename to backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/Mapper.java index 095921e458..a120d7d9ff 100644 --- a/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/services/Mapper.java +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/Mapper.java @@ -1,8 +1,8 @@ -package co.airy.core.sources.facebook.services; +package co.airy.core.sources.facebook.api; import co.airy.avro.communication.Message; -import co.airy.core.sources.facebook.model.SendMessagePayload; -import co.airy.core.sources.facebook.model.SendMessageRequest; +import co.airy.core.sources.facebook.api.model.SendMessagePayload; +import co.airy.core.sources.facebook.dto.SendMessageRequest; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.stereotype.Service; @@ -28,7 +28,7 @@ public SendMessagePayload fromSendMessageRequest(SendMessageRequest sendMessageR SendMessagePayload.SendMessagePayloadBuilder builder = SendMessagePayload.builder() .recipient(SendMessagePayload.MessageRecipient.builder() - .id(sendMessageRequest.getSourceConversationId()) + .id(sendMessageRequest.getConversation().getSourceConversationId()) .build()) .message(messagePayload); diff --git a/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/LongLivingUserAccessToken.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/LongLivingUserAccessToken.java new file mode 100644 index 0000000000..193b7039dd --- /dev/null +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/LongLivingUserAccessToken.java @@ -0,0 +1,14 @@ +package co.airy.core.sources.facebook.api.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class LongLivingUserAccessToken { + private String accessToken; +} + diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/facebook/FbPageWithConnectInfo.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/PageWithConnectInfo.java similarity index 91% rename from backend/api/admin/src/main/java/co/airy/core/api/admin/sources/facebook/FbPageWithConnectInfo.java rename to backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/PageWithConnectInfo.java index 62d9a9f328..8416e4ceb3 100644 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/sources/facebook/FbPageWithConnectInfo.java +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/PageWithConnectInfo.java @@ -1,4 +1,4 @@ -package co.airy.core.api.admin.sources.facebook; +package co.airy.core.sources.facebook.api.model; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; @@ -12,7 +12,7 @@ @AllArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_EMPTY) -public class FbPageWithConnectInfo { +public class PageWithConnectInfo { private String id; private String nameWithLocationDescriptor; private String accessToken; diff --git a/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/Pages.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/Pages.java new file mode 100644 index 0000000000..743883e729 --- /dev/null +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/Pages.java @@ -0,0 +1,23 @@ +package co.airy.core.sources.facebook.api.model; + +import co.airy.core.sources.facebook.api.model.PageWithConnectInfo; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Pages { + private List data; + private Paging paging; + + @Data + public static class Paging { + private String next; + } +} + diff --git a/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/Participants.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/Participants.java new file mode 100644 index 0000000000..4a4e79c954 --- /dev/null +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/Participants.java @@ -0,0 +1,38 @@ +package co.airy.core.sources.facebook.api.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Participants { + private List data; + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class Entry { + private Participant participants; + private String id; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class Participant { + private List data; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class ParticipantEntry { + private String name; + private String email; + private String id; + } +} diff --git a/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/model/SendMessagePayload.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/SendMessagePayload.java similarity index 95% rename from backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/model/SendMessagePayload.java rename to backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/SendMessagePayload.java index 701e795d1f..48e8f0c850 100644 --- a/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/model/SendMessagePayload.java +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/SendMessagePayload.java @@ -1,4 +1,4 @@ -package co.airy.core.sources.facebook.model; +package co.airy.core.sources.facebook.api.model; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/UserProfile.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/UserProfile.java new file mode 100644 index 0000000000..4e46405183 --- /dev/null +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/api/model/UserProfile.java @@ -0,0 +1,22 @@ +package co.airy.core.sources.facebook.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class UserProfile { + @JsonProperty("first_name") + private String firstName; + @JsonProperty("last_name") + private String lastName; + @JsonProperty("profile_pic") + private String profilePic; + @JsonProperty("locale") + private String locale; +} diff --git a/backend/sources/twilio/sender/src/main/java/co/airy/core/sources/twilio/model/SendMessageRequest.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/dto/Conversation.java similarity index 64% rename from backend/sources/twilio/sender/src/main/java/co/airy/core/sources/twilio/model/SendMessageRequest.java rename to backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/dto/Conversation.java index 68a157a884..02b9654b57 100644 --- a/backend/sources/twilio/sender/src/main/java/co/airy/core/sources/twilio/model/SendMessageRequest.java +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/dto/Conversation.java @@ -1,21 +1,21 @@ -package co.airy.core.sources.twilio.model; +package co.airy.core.sources.facebook.dto; import co.airy.avro.communication.Channel; -import co.airy.avro.communication.Message; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import java.io.Serializable; +import java.util.Map; @Data -@Builder +@Builder(toBuilder = true) @NoArgsConstructor @AllArgsConstructor -public class SendMessageRequest implements Serializable { +public class Conversation implements Serializable { private String sourceConversationId; private String channelId; - private Message message; private Channel channel; + private Map metadata; } diff --git a/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/dto/SendMessageRequest.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/dto/SendMessageRequest.java new file mode 100644 index 0000000000..b23316beb6 --- /dev/null +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/dto/SendMessageRequest.java @@ -0,0 +1,18 @@ +package co.airy.core.sources.facebook.dto; + +import co.airy.avro.communication.Message; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SendMessageRequest implements Serializable { + private Conversation conversation; + private Message message; +} diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/AvailableChannelsRequestPayload.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/payload/ConnectRequestPayload.java similarity index 51% rename from backend/api/admin/src/main/java/co/airy/core/api/admin/payload/AvailableChannelsRequestPayload.java rename to backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/payload/ConnectRequestPayload.java index e14fb78fe2..45e5a752b4 100644 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/AvailableChannelsRequestPayload.java +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/payload/ConnectRequestPayload.java @@ -1,4 +1,4 @@ -package co.airy.core.api.admin.payload; +package co.airy.core.sources.facebook.payload; import lombok.AllArgsConstructor; import lombok.Data; @@ -9,9 +9,11 @@ @Data @NoArgsConstructor @AllArgsConstructor -public class AvailableChannelsRequestPayload { +public class ConnectRequestPayload { @NotNull - String source; + private String pageId; @NotNull - String token; + private String pageToken; + private String name; + private String imageUrl; } diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/DisconnectChannelRequestPayload.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/payload/ExploreRequestPayload.java similarity index 60% rename from backend/api/admin/src/main/java/co/airy/core/api/admin/payload/DisconnectChannelRequestPayload.java rename to backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/payload/ExploreRequestPayload.java index 7b1e4fc4bf..f9182fb6f9 100644 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/DisconnectChannelRequestPayload.java +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/payload/ExploreRequestPayload.java @@ -1,16 +1,15 @@ -package co.airy.core.api.admin.payload; +package co.airy.core.sources.facebook.payload; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import javax.validation.constraints.NotNull; -import java.util.UUID; @Data @NoArgsConstructor @AllArgsConstructor -public class DisconnectChannelRequestPayload { +public class ExploreRequestPayload { @NotNull - UUID channelId; + String authToken; } diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/AvailableChannelsResponsePayload.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/payload/ExploreResponsePayload.java similarity index 54% rename from backend/api/admin/src/main/java/co/airy/core/api/admin/payload/AvailableChannelsResponsePayload.java rename to backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/payload/ExploreResponsePayload.java index 7915c84736..5cd928b9db 100644 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/AvailableChannelsResponsePayload.java +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/payload/ExploreResponsePayload.java @@ -1,4 +1,4 @@ -package co.airy.core.api.admin.payload; +package co.airy.core.sources.facebook.payload; import lombok.AllArgsConstructor; import lombok.Data; @@ -9,6 +9,7 @@ @Data @NoArgsConstructor @AllArgsConstructor -public class AvailableChannelsResponsePayload { - private List data; +public class ExploreResponsePayload { + private List data; } + diff --git a/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/AvailableChannelPayload.java b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/payload/PageInfoResponsePayload.java similarity index 68% rename from backend/api/admin/src/main/java/co/airy/core/api/admin/payload/AvailableChannelPayload.java rename to backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/payload/PageInfoResponsePayload.java index 0a4b9a91d3..8b776dc13f 100644 --- a/backend/api/admin/src/main/java/co/airy/core/api/admin/payload/AvailableChannelPayload.java +++ b/backend/sources/facebook/connector/src/main/java/co/airy/core/sources/facebook/payload/PageInfoResponsePayload.java @@ -1,4 +1,4 @@ -package co.airy.core.api.admin.payload; +package co.airy.core.sources.facebook.payload; import lombok.AllArgsConstructor; import lombok.Builder; @@ -9,8 +9,8 @@ @Builder @NoArgsConstructor @AllArgsConstructor -public class AvailableChannelPayload { - private String sourceChannelId; +public class PageInfoResponsePayload { + private String pageId; private String name; private String imageUrl; private boolean connected; diff --git a/backend/sources/facebook/connector/src/main/resources/application.properties b/backend/sources/facebook/connector/src/main/resources/application.properties new file mode 100644 index 0000000000..fe8ecd4e38 --- /dev/null +++ b/backend/sources/facebook/connector/src/main/resources/application.properties @@ -0,0 +1,7 @@ +auth.jwt-secret=${JWT_SECRET} +facebook.webhook-secret=${FACEBOOK_WEBHOOK_SECRET} +kafka.brokers=${KAFKA_BROKERS} +kafka.cleanup=${KAFKA_CLEANUP:false} +kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS} +kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} +kafka.suppress-interval-ms=${KAFKA_SUPPRESS_INTERVAL_MS:3000} diff --git a/backend/sources/facebook/connector/src/test/java/co/airy/core/sources/facebook/FetchMetadataTest.java b/backend/sources/facebook/connector/src/test/java/co/airy/core/sources/facebook/FetchMetadataTest.java new file mode 100644 index 0000000000..48462b76cc --- /dev/null +++ b/backend/sources/facebook/connector/src/test/java/co/airy/core/sources/facebook/FetchMetadataTest.java @@ -0,0 +1,144 @@ +package co.airy.core.sources.facebook; + +import co.airy.avro.communication.Channel; +import co.airy.avro.communication.ChannelConnectionState; +import co.airy.avro.communication.DeliveryState; +import co.airy.avro.communication.Message; +import co.airy.avro.communication.Metadata; +import co.airy.model.metadata.MetadataKeys; +import co.airy.avro.communication.SenderType; +import co.airy.core.sources.facebook.api.Api; +import co.airy.core.sources.facebook.api.model.UserProfile; +import co.airy.kafka.schema.Topic; +import co.airy.kafka.schema.application.ApplicationCommunicationChannels; +import co.airy.kafka.schema.application.ApplicationCommunicationMessages; +import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; +import co.airy.kafka.test.KafkaTestHelper; +import co.airy.kafka.test.junit.SharedKafkaTestResource; +import co.airy.spring.core.AirySpringBootApplication; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import java.time.Instant; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@SpringBootTest(classes = AirySpringBootApplication.class) +@TestPropertySource(value = "classpath:test.properties") +@ExtendWith(SpringExtension.class) +class FetchMetadataTest { + + @RegisterExtension + public static final SharedKafkaTestResource sharedKafkaTestResource = new SharedKafkaTestResource(); + private static KafkaTestHelper kafkaTestHelper; + + private static final Topic applicationCommunicationChannels = new ApplicationCommunicationChannels(); + private static final Topic applicationCommunicationMessages = new ApplicationCommunicationMessages(); + private static final Topic applicationCommunicationMetadata = new ApplicationCommunicationMetadata(); + + @MockBean + private Api api; + + @Autowired + @InjectMocks + private Connector worker; + + @BeforeAll + static void beforeAll() throws Exception { + kafkaTestHelper = new KafkaTestHelper(sharedKafkaTestResource, + applicationCommunicationChannels, + applicationCommunicationMessages, + applicationCommunicationMetadata + ); + + kafkaTestHelper.beforeAll(); + } + + @AfterAll + static void afterAll() throws Exception { + kafkaTestHelper.afterAll(); + } + + @BeforeEach + void beforeEach() throws InterruptedException { + MockitoAnnotations.initMocks(this); + } + + @Test + void canFetchMetadata() throws Exception { + final String sourceConversationId = "source-conversation-id"; + final String channelId = "channel-id"; + final String token = "token"; + + final String firstName = "Grace"; + final String lastName = "Grace"; + final String avatarUrl = "http://placehold.it/120x120&text=image1"; + + UserProfile userProfile = new UserProfile(firstName, lastName, avatarUrl, "en"); + ArgumentCaptor sourceConversationIdCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor tokenCaptor = ArgumentCaptor.forClass(String.class); + Mockito.when(api.getProfileFromContact(sourceConversationIdCaptor.capture(), tokenCaptor.capture())) + .thenReturn(userProfile); + + kafkaTestHelper.produceRecords(List.of( + new ProducerRecord<>(applicationCommunicationChannels.name(), channelId, Channel.newBuilder() + .setToken(token) + .setSourceChannelId("ps-id") + .setSource("facebook") + .setName("name") + .setId(channelId) + .setConnectionState(ChannelConnectionState.CONNECTED) + .build() + ), + new ProducerRecord<>(applicationCommunicationMessages.name(), "other-message-id", + Message.newBuilder() + .setId("other-message-id") + .setSource("facebook") + .setSentAt(Instant.now().toEpochMilli()) + .setSenderId(sourceConversationId) + .setSenderType(SenderType.SOURCE_CONTACT) + .setDeliveryState(DeliveryState.DELIVERED) + .setConversationId("conversationId") + .setChannelId(channelId) + .setContent("{\"text\":\"hello world\"}") + .build()) + )); + + List metadataList = kafkaTestHelper.consumeValues(4, applicationCommunicationMetadata.name()); + assertThat(metadataList, hasSize(4)); + + assertTrue(metadataList.stream().anyMatch((metadata -> + metadata.getKey().equals(MetadataKeys.Source.Contact.FIRST_NAME) && metadata.getValue().equals(firstName) + ))); + assertTrue(metadataList.stream().anyMatch((metadata -> + metadata.getKey().equals(MetadataKeys.Source.Contact.LAST_NAME) && metadata.getValue().equals(lastName) + ))); + assertTrue(metadataList.stream().anyMatch((metadata -> + metadata.getKey().equals(MetadataKeys.Source.Contact.AVATAR_URL) && metadata.getValue().equals(avatarUrl) + ))); + assertTrue(metadataList.stream().anyMatch((metadata -> + metadata.getKey().equals(MetadataKeys.Source.Contact.FETCH_STATE) && metadata.getValue().equals("ok") + ))); + + assertThat(sourceConversationIdCaptor.getValue(), equalTo(sourceConversationId)); + assertThat(tokenCaptor.getValue(), equalTo(token)); + } +} diff --git a/backend/sources/facebook/sender/src/test/java/co/airy/core/sources/facebook/SendMessageTest.java b/backend/sources/facebook/connector/src/test/java/co/airy/core/sources/facebook/SendMessageTest.java similarity index 88% rename from backend/sources/facebook/sender/src/test/java/co/airy/core/sources/facebook/SendMessageTest.java rename to backend/sources/facebook/connector/src/test/java/co/airy/core/sources/facebook/SendMessageTest.java index e9aae84afc..a0a9e9b08d 100644 --- a/backend/sources/facebook/sender/src/test/java/co/airy/core/sources/facebook/SendMessageTest.java +++ b/backend/sources/facebook/connector/src/test/java/co/airy/core/sources/facebook/SendMessageTest.java @@ -5,11 +5,12 @@ import co.airy.avro.communication.DeliveryState; import co.airy.avro.communication.Message; import co.airy.avro.communication.SenderType; -import co.airy.core.sources.facebook.model.SendMessagePayload; -import co.airy.core.sources.facebook.services.Api; +import co.airy.core.sources.facebook.api.model.SendMessagePayload; +import co.airy.core.sources.facebook.api.Api; import co.airy.kafka.schema.Topic; import co.airy.kafka.schema.application.ApplicationCommunicationChannels; import co.airy.kafka.schema.application.ApplicationCommunicationMessages; +import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; import co.airy.kafka.test.KafkaTestHelper; import co.airy.kafka.test.junit.SharedKafkaTestResource; import co.airy.spring.core.AirySpringBootApplication; @@ -26,6 +27,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import java.time.Instant; @@ -39,11 +41,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.doNothing; -@SpringBootTest(properties = { - "kafka.cleanup=true", - "kafka.commit-interval-ms=100", - "facebook.app-id=12345" -}, classes = AirySpringBootApplication.class) +@SpringBootTest(classes = AirySpringBootApplication.class) +@TestPropertySource(value = "classpath:test.properties") @ExtendWith(SpringExtension.class) class SendMessageTest { @@ -53,21 +52,24 @@ class SendMessageTest { private static final Topic applicationCommunicationChannels = new ApplicationCommunicationChannels(); private static final Topic applicationCommunicationMessages = new ApplicationCommunicationMessages(); - - @MockBean - private Api api; + private static final Topic applicationCommunicationMetadata = new ApplicationCommunicationMetadata(); @Autowired @InjectMocks - private Sender worker; + private Connector connector; + + @Autowired + private Stores stores; - private static boolean streamInitialized = false; + @MockBean + private Api api; @BeforeAll static void beforeAll() throws Exception { kafkaTestHelper = new KafkaTestHelper(sharedKafkaTestResource, applicationCommunicationChannels, - applicationCommunicationMessages + applicationCommunicationMessages, + applicationCommunicationMetadata ); kafkaTestHelper.beforeAll(); @@ -81,12 +83,7 @@ static void afterAll() throws Exception { @BeforeEach void beforeEach() throws InterruptedException { MockitoAnnotations.initMocks(this); - - if (!streamInitialized) { - retryOnException(() -> assertEquals(worker.getStreamState(), RUNNING), "Failed to reach RUNNING state."); - - streamInitialized = true; - } + retryOnException(() -> assertEquals(stores.getStreamState(), RUNNING), "Failed to reach RUNNING state."); } @Test @@ -144,7 +141,6 @@ void canSendMessageViaTheFacebookApi() throws Exception { retryOnException(() -> { final SendMessagePayload sendMessagePayload = payloadCaptor.getValue(); - assertThat(sendMessagePayload.getRecipient().getId(), equalTo(sourceConversationId)); assertThat(sendMessagePayload.getMessage().getText(), equalTo(text)); diff --git a/backend/sources/facebook/webhook/src/test/java/co/airy/core/sources/facebook/FacebookWebhookTest.java b/backend/sources/facebook/connector/src/test/java/co/airy/core/sources/facebook/WebhookControllerTest.java similarity index 83% rename from backend/sources/facebook/webhook/src/test/java/co/airy/core/sources/facebook/FacebookWebhookTest.java rename to backend/sources/facebook/connector/src/test/java/co/airy/core/sources/facebook/WebhookControllerTest.java index 634d510a03..44b65987f9 100644 --- a/backend/sources/facebook/webhook/src/test/java/co/airy/core/sources/facebook/FacebookWebhookTest.java +++ b/backend/sources/facebook/connector/src/test/java/co/airy/core/sources/facebook/WebhookControllerTest.java @@ -13,6 +13,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; @@ -24,14 +25,11 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -@SpringBootTest( - properties = { - "facebook.webhook-secret=theansweris42" - }, - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = AirySpringBootApplication.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = AirySpringBootApplication.class) +@TestPropertySource(value = "classpath:test.properties") @AutoConfigureMockMvc @ExtendWith(SpringExtension.class) -class FacebookWebhookTest { +class WebhookControllerTest { @RegisterExtension public static final SharedKafkaTestResource sharedKafkaTestResource = new SharedKafkaTestResource(); @@ -49,20 +47,18 @@ static void beforeAll() throws Exception { kafkaTestHelper.beforeAll(); } + @AfterAll + static void afterAll() throws Exception { + kafkaTestHelper.afterAll(); + } + @Test - void returns200() throws Exception { - mvc.perform(post("/facebook") - .content("whatever")) - .andExpect(status().isOk()); + void canAcceptAnything() throws Exception { + mvc.perform(post("/facebook").content("whatever")).andExpect(status().isOk()); List records = kafkaTestHelper.consumeValues(1, sourceFacebookEvents.name()); assertThat(records, hasSize(1)); assertEquals("whatever", records.get(0)); } - - @AfterAll - static void afterAll() throws Exception { - kafkaTestHelper.afterAll(); - } } diff --git a/backend/sources/facebook/connector/src/test/resources/test.properties b/backend/sources/facebook/connector/src/test/resources/test.properties new file mode 100644 index 0000000000..eaa17ecac0 --- /dev/null +++ b/backend/sources/facebook/connector/src/test/resources/test.properties @@ -0,0 +1,7 @@ +kafka.cleanup=true +kafka.commit-interval-ms=100 +kafka.suppress-interval-ms=0 +facebook.webhook-secret=theansweris42 +facebook.app-id=12345 +facebook.app-secret=secret +auth.jwt-secret=42424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242 \ No newline at end of file diff --git a/backend/sources/facebook/events-router/BUILD b/backend/sources/facebook/events-router/BUILD index ad407f81f8..b1717175e2 100644 --- a/backend/sources/facebook/events-router/BUILD +++ b/backend/sources/facebook/events-router/BUILD @@ -4,10 +4,9 @@ load("//tools/build:container_push.bzl", "container_push") app_deps = [ "//backend:base_app", - "//backend:channel", - "//backend:message", + "//backend/model/channel:channel", + "//backend/model/message:message", "//lib/java/uuid", - "//lib/java/payload", "//lib/java/log", "//lib/java/kafka/schema:source-facebook-events", "//lib/java/spring/kafka/core:spring-kafka-core", diff --git a/backend/sources/facebook/events-router/src/main/resources/application.properties b/backend/sources/facebook/events-router/src/main/resources/application.properties index ceafe31aa6..67867e2b41 100644 --- a/backend/sources/facebook/events-router/src/main/resources/application.properties +++ b/backend/sources/facebook/events-router/src/main/resources/application.properties @@ -1,5 +1,5 @@ kafka.brokers=${KAFKA_BROKERS} kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} kafka.cleanup=${KAFKA_CLEANUP:false} -kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS:30000} +kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS} facebook.app-id=${FACEBOOK_APP_ID} diff --git a/backend/sources/facebook/events-router/src/test/java/co/airy/core/sources/facebook/EventsRouterTest.java b/backend/sources/facebook/events-router/src/test/java/co/airy/core/sources/facebook/EventsRouterTest.java index a82b2a7c93..532530faf7 100644 --- a/backend/sources/facebook/events-router/src/test/java/co/airy/core/sources/facebook/EventsRouterTest.java +++ b/backend/sources/facebook/events-router/src/test/java/co/airy/core/sources/facebook/EventsRouterTest.java @@ -20,6 +20,7 @@ import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import java.util.ArrayList; @@ -38,11 +39,8 @@ import static org.hamcrest.collection.IsCollectionWithSize.hasSize; import static org.junit.jupiter.api.Assertions.assertEquals; -@SpringBootTest(properties = { - "kafka.cleanup=true", - "kafka.commit-interval-ms=100", - "facebook.app-id=12345" -}, classes = AirySpringBootApplication.class) +@SpringBootTest(classes = AirySpringBootApplication.class) +@TestPropertySource(value = "classpath:test.properties") @ExtendWith(SpringExtension.class) class EventsRouterTest { @@ -57,8 +55,6 @@ class EventsRouterTest { @Autowired private EventsRouter worker; - private static boolean streamInitialized = false; - @BeforeAll static void beforeAll() throws Exception { kafkaTestHelper = new KafkaTestHelper(sharedKafkaTestResource, @@ -77,11 +73,7 @@ static void afterAll() throws Exception { @BeforeEach void beforeEach() throws InterruptedException { - if (!streamInitialized) { - retryOnException(() -> assertEquals(worker.getStreamState(), RUNNING), "Failed to reach RUNNING state."); - - streamInitialized = true; - } + retryOnException(() -> assertEquals(worker.getStreamState(), RUNNING), "Failed to reach RUNNING state."); } private final String eventTemplate = "{\"object\":\"page\",\"entry\":[{\"id\":\"%s\",\"time\":1550050754198," + @@ -140,8 +132,8 @@ void joinsAndCountsMessagesCorrectly() throws Exception { List messages = kafkaTestHelper.consumeValues(totalMessages, applicationCommunicationMessages.name()); assertThat(messages, hasSize(totalMessages)); - messagesPerContact.forEach((conversationId, expectedCount) -> { - assertEquals(messages.stream().filter(m -> m.getConversationId().equals(conversationId)).count(), expectedCount.longValue()); - }); + messagesPerContact.forEach((conversationId, expectedCount) -> + assertEquals(messages.stream().filter(m -> m.getConversationId().equals(conversationId)).count(), expectedCount.longValue()) + ); } } diff --git a/backend/sources/facebook/events-router/src/test/resources/test.properties b/backend/sources/facebook/events-router/src/test/resources/test.properties new file mode 100644 index 0000000000..0a30b9f6c9 --- /dev/null +++ b/backend/sources/facebook/events-router/src/test/resources/test.properties @@ -0,0 +1,3 @@ +kafka.cleanup=true +kafka.commit-interval-ms=100 +facebook.app-id=12345 \ No newline at end of file diff --git a/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/Sender.java b/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/Sender.java deleted file mode 100644 index 9f39d30cc8..0000000000 --- a/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/Sender.java +++ /dev/null @@ -1,120 +0,0 @@ -package co.airy.core.sources.facebook; - -import co.airy.avro.communication.Channel; -import co.airy.avro.communication.ChannelConnectionState; -import co.airy.avro.communication.DeliveryState; -import co.airy.avro.communication.Message; -import co.airy.avro.communication.SenderType; -import co.airy.core.sources.facebook.model.SendMessagePayload; -import co.airy.core.sources.facebook.model.SendMessageRequest; -import co.airy.core.sources.facebook.services.Api; -import co.airy.core.sources.facebook.services.Mapper; -import co.airy.kafka.schema.application.ApplicationCommunicationChannels; -import co.airy.kafka.schema.application.ApplicationCommunicationMessages; -import co.airy.kafka.streams.KafkaStreamsWrapper; -import co.airy.log.AiryLoggerFactory; -import org.apache.kafka.streams.KafkaStreams; -import org.apache.kafka.streams.StreamsBuilder; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.KTable; -import org.slf4j.Logger; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.stereotype.Component; - -import static co.airy.avro.communication.MessageRepository.updateDeliveryState; - -@Component -public class Sender implements DisposableBean, ApplicationListener { - private static final Logger log = AiryLoggerFactory.getLogger(Sender.class); - private static final String appId = "sources.facebook.Sender"; - - private final KafkaStreamsWrapper streams; - private final Api api; - private final Mapper mapper; - - Sender(KafkaStreamsWrapper streams, Api api, Mapper mapper) { - this.streams = streams; - this.api = api; - this.mapper = mapper; - } - - public void startStream() { - final StreamsBuilder builder = new StreamsBuilder(); - - // Channels table - KTable channelsTable = builder.table(new ApplicationCommunicationChannels().name()) - .filter((sourceChannelId, channel) -> "facebook".equalsIgnoreCase(channel.getSource()) - && channel.getConnectionState().equals(ChannelConnectionState.CONNECTED)); - - final KStream messageStream = builder.stream(new ApplicationCommunicationMessages().name()) - .filter((messageId, message) -> "facebook".equalsIgnoreCase(message.getSource())) - .selectKey((messageId, message) -> message.getConversationId()); - - final KTable contextTable = messageStream - .groupByKey() - .aggregate(SendMessageRequest::new, - (conversationId, message, aggregate) -> { - if (SenderType.SOURCE_CONTACT.equals(message.getSenderType())) { - aggregate.setSourceConversationId(message.getSenderId()); - } - - aggregate.setChannelId(message.getChannelId()); - - return aggregate; - }) - .join(channelsTable, SendMessageRequest::getChannelId, (aggregate, channel) -> { - aggregate.setChannel(channel); - return aggregate; - }); - - messageStream.filter((messageId, message) -> DeliveryState.PENDING.equals(message.getDeliveryState())) - .join(contextTable, (message, sendMessageRequest) -> { - sendMessageRequest.setMessage(message); - return sendMessageRequest; - }) - .mapValues(this::sendMessage) - .to(new ApplicationCommunicationMessages().name()); - - streams.start(builder.build(), appId); - } - - private Message sendMessage(SendMessageRequest sendMessageRequest) { - final Message message = sendMessageRequest.getMessage(); - - try { - final String pageToken = sendMessageRequest.getChannel().getToken(); - final SendMessagePayload fbSendMessagePayload = mapper.fromSendMessageRequest(sendMessageRequest); - - api.sendMessage(pageToken, fbSendMessagePayload); - - updateDeliveryState(message, DeliveryState.DELIVERED); - return message; - } catch (ApiException e) { - log.error(String.format("Failed to send a message to Facebook \n SendMessageRequest: %s \n Error Message: %s \n", sendMessageRequest, e.getMessage()), e); - } catch (Exception e) { - log.error(String.format("Failed to send a message to Facebook \n SendMessageRequest: %s", sendMessageRequest), e); - } - - updateDeliveryState(message, DeliveryState.FAILED); - return message; - } - - @Override - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - startStream(); - } - - @Override - public void destroy() { - if (streams != null) { - streams.close(); - } - } - - // visible for testing - KafkaStreams.State getStreamState() { - return streams.state(); - } -} diff --git a/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/services/Api.java b/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/services/Api.java deleted file mode 100644 index 617b3a6968..0000000000 --- a/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/services/Api.java +++ /dev/null @@ -1,85 +0,0 @@ -package co.airy.core.sources.facebook.services; - -import co.airy.core.sources.facebook.ApiException; -import co.airy.core.sources.facebook.model.SendMessagePayload; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.context.ApplicationListener; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.client.ClientHttpResponse; -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; -import org.springframework.stereotype.Service; -import org.springframework.web.client.ResponseErrorHandler; -import org.springframework.web.client.RestTemplate; - -import java.io.IOException; - -/* - @see https://developers.facebook.com/docs/messenger-platform/reference/send-api/ - */ -@Service -public class Api implements ApplicationListener { - private final RestTemplateBuilder restTemplateBuilder; - private final ObjectMapper objectMapper; - - private RestTemplate restTemplate; - - private static final String requestTemplate = "https://graph.facebook.com/v3.2/me/messages?access_token=%s"; - - private final HttpHeaders httpHeaders = new HttpHeaders(); - - private static final String errorMessageTemplate = - "Exception while sending a message to Facebook: \n" + - "Http Status Code: %s \n" + - "Error Message: %s \n"; - - public Api(ObjectMapper objectMapper, RestTemplateBuilder restTemplateBuilder) { - httpHeaders.setContentType(MediaType.APPLICATION_JSON); - this.objectMapper = objectMapper; - this.restTemplateBuilder = restTemplateBuilder; - } - - - public void sendMessage(final String pageToken, SendMessagePayload sendMessagePayload) { - String fbReqUrl = String.format(requestTemplate, pageToken); - - restTemplate.postForEntity(fbReqUrl, new HttpEntity<>(sendMessagePayload, httpHeaders), FbSendMessageResponse.class); - } - - @Override - public void onApplicationEvent(ApplicationReadyEvent event) { - restTemplate = restTemplateBuilder - .errorHandler(new ResponseErrorHandler() { - @Override - public boolean hasError(ClientHttpResponse response) throws IOException { - return response.getRawStatusCode() != HttpStatus.OK.value(); - } - - @Override - public void handleError(ClientHttpResponse response) throws IOException { - throw new ApiException(String.format(errorMessageTemplate, response.getRawStatusCode(), new String(response.getBody().readAllBytes()))); - } - }) - .additionalMessageConverters(new MappingJackson2HttpMessageConverter(objectMapper)) - .build(); - } - - @Data - @NoArgsConstructor - @AllArgsConstructor - private static class FbSendMessageResponse { - @JsonProperty("recipient_id") - private String recipientId; - - @JsonProperty("message_id") - private String messageId; - } -} diff --git a/backend/sources/facebook/sender/src/main/resources/application.properties b/backend/sources/facebook/sender/src/main/resources/application.properties deleted file mode 100644 index 98d0f6de52..0000000000 --- a/backend/sources/facebook/sender/src/main/resources/application.properties +++ /dev/null @@ -1,4 +0,0 @@ -kafka.brokers=${KAFKA_BROKERS} -kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} -kafka.cleanup=${KAFKA_CLEANUP:false} -kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS:30000} diff --git a/backend/sources/facebook/webhook/BUILD b/backend/sources/facebook/webhook/BUILD deleted file mode 100644 index 9b87d117f8..0000000000 --- a/backend/sources/facebook/webhook/BUILD +++ /dev/null @@ -1,37 +0,0 @@ -load("//tools/build:springboot.bzl", "springboot") -load("//tools/build:junit5.bzl", "junit5") -load("//tools/build:container_push.bzl", "container_push") - -app_deps = [ - "//backend:base_app", - "//:springboot_actuator", - "//lib/java/kafka/schema:source-facebook-events", - "//lib/java/spring/kafka/core:spring-kafka-core", - "//lib/java/spring/kafka/healthcheck", -] - -springboot( - name = "webhook", - srcs = glob(["src/main/java/**/*.java"]), - main_class = "co.airy.spring.core.AirySpringBootApplication", - deps = app_deps, -) - -[ - junit5( - size = "small", - file = file, - resources = glob(["src/test/resources/**/*"]), - deps = [ - ":app", - "//backend:base_test", - "//lib/java/kafka/test:kafka-test", - ] + app_deps, - ) - for file in glob(["src/test/java/**/*Test.java"]) -] - -container_push( - registry = "ghcr.io/airyhq/sources", - repository = "facebook-webhook", -) diff --git a/backend/sources/facebook/webhook/src/main/resources/application.properties b/backend/sources/facebook/webhook/src/main/resources/application.properties deleted file mode 100644 index 96c9aeff81..0000000000 --- a/backend/sources/facebook/webhook/src/main/resources/application.properties +++ /dev/null @@ -1,4 +0,0 @@ -kafka.brokers=${KAFKA_BROKERS} -kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} - -facebook.webhook-secret=${FACEBOOK_WEBHOOK_SECRET} diff --git a/backend/sources/google/sender/BUILD b/backend/sources/google/connector/BUILD similarity index 74% rename from backend/sources/google/sender/BUILD rename to backend/sources/google/connector/BUILD index e81073a849..0e3d38f89d 100644 --- a/backend/sources/google/sender/BUILD +++ b/backend/sources/google/connector/BUILD @@ -4,14 +4,19 @@ load("//tools/build:container_push.bzl", "container_push") app_deps = [ "//backend:base_app", - "//backend:message", + "//backend/model/channel:channel", + "//backend/model/message:message", "//lib/java/spring/kafka/core:spring-kafka-core", "//lib/java/spring/kafka/streams:spring-kafka-streams", + "//lib/java/uuid", + "//lib/java/kafka/schema:source-google-events", + "//lib/java/spring/web:spring-web", + "//lib/java/spring/auth:spring-auth", "@maven//:com_google_auth_google_auth_library_oauth2_http", ] springboot( - name = "google-sender", + name = "google-connector", srcs = glob(["src/main/java/**/*.java"]), main_class = "co.airy.spring.core.AirySpringBootApplication", deps = app_deps, @@ -33,5 +38,5 @@ springboot( container_push( registry = "ghcr.io/airyhq/sources", - repository = "google-sender", + repository = "google-connector", ) diff --git a/backend/sources/google/sender/src/main/java/co/airy/core/sources/google/ApiException.java b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/ApiException.java similarity index 75% rename from backend/sources/google/sender/src/main/java/co/airy/core/sources/google/ApiException.java rename to backend/sources/google/connector/src/main/java/co/airy/core/sources/google/ApiException.java index bf0a892d23..d112f46a0c 100644 --- a/backend/sources/google/sender/src/main/java/co/airy/core/sources/google/ApiException.java +++ b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/ApiException.java @@ -1,9 +1,6 @@ package co.airy.core.sources.google; public class ApiException extends RuntimeException { - public ApiException() { - super(); - } public ApiException(String msg) { super(msg); } diff --git a/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/ChannelsController.java b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/ChannelsController.java new file mode 100644 index 0000000000..6fedf6398b --- /dev/null +++ b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/ChannelsController.java @@ -0,0 +1,108 @@ +package co.airy.core.sources.google; + +import co.airy.avro.communication.Channel; +import co.airy.avro.communication.ChannelConnectionState; +import co.airy.kafka.schema.application.ApplicationCommunicationChannels; +import co.airy.spring.web.payload.EmptyResponsePayload; +import co.airy.uuid.UUIDv5; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import javax.validation.Valid; +import javax.validation.constraints.NotNull; +import java.util.UUID; + +import static co.airy.model.channel.ChannelPayload.fromChannel; + +@RestController +public class ChannelsController { + private static final String applicationCommunicationChannels = new ApplicationCommunicationChannels().name(); + + private final Stores stores; + private final KafkaProducer producer; + + public ChannelsController(Stores stores, KafkaProducer producer) { + this.stores = stores; + this.producer = producer; + } + + @PostMapping("/google.connect") + ResponseEntity connect(@RequestBody @Valid ConnectChannelRequestPayload requestPayload) { + final String gbmId = requestPayload.getGbmId(); + final String sourceIdentifier = "google"; + + final String channelId = UUIDv5.fromNamespaceAndName(sourceIdentifier, gbmId).toString(); + + final Channel channel = Channel.newBuilder() + .setId(channelId) + .setConnectionState(ChannelConnectionState.CONNECTED) + .setSource(sourceIdentifier) + .setSourceChannelId(gbmId) + .setName(requestPayload.getName()) + .setImageUrl(requestPayload.getImageUrl()) + .build(); + + try { + producer.send(new ProducerRecord<>(applicationCommunicationChannels, channel.getId(), channel)).get(); + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build(); + } + + return ResponseEntity.ok(fromChannel(channel)); + } + + @PostMapping("/google.disconnect") + ResponseEntity disconnect(@RequestBody @Valid DisconnectChannelRequestPayload requestPayload) { + final String channelId = requestPayload.getChannelId().toString(); + + final Channel channel = stores.getChannelsStore().get(channelId); + + if (channel == null) { + return ResponseEntity.notFound().build(); + } + + if (channel.getConnectionState().equals(ChannelConnectionState.DISCONNECTED)) { + return ResponseEntity.accepted().build(); + } + + channel.setConnectionState(ChannelConnectionState.DISCONNECTED); + channel.setToken(null); + + try { + producer.send(new ProducerRecord<>(applicationCommunicationChannels, channel.getId(), channel)).get(); + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build(); + } + + return ResponseEntity.ok(new EmptyResponsePayload()); + } +} + +@Data +@NoArgsConstructor +@AllArgsConstructor +class ConnectChannelRequestPayload { + @NotNull + private String gbmId; + + @NotNull + private String name; + + private String imageUrl; +} + +@Data +@NoArgsConstructor +@AllArgsConstructor +class DisconnectChannelRequestPayload { + @NotNull + private UUID channelId; +} diff --git a/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/Connector.java b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/Connector.java new file mode 100644 index 0000000000..fcaef49dbc --- /dev/null +++ b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/Connector.java @@ -0,0 +1,61 @@ +package co.airy.core.sources.google; + +import co.airy.avro.communication.DeliveryState; +import co.airy.avro.communication.Message; +import co.airy.core.sources.google.model.SendMessagePayload; +import co.airy.core.sources.google.model.SendMessageRequest; +import co.airy.core.sources.google.services.Api; +import co.airy.core.sources.google.services.Mapper; +import co.airy.log.AiryLoggerFactory; +import co.airy.spring.auth.IgnoreAuthPattern; +import co.airy.spring.web.filters.RequestLoggingIgnorePatterns; +import org.slf4j.Logger; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +import java.util.List; + +import static co.airy.model.message.MessageRepository.updateDeliveryState; + +@Component +public class Connector { + private static final Logger log = AiryLoggerFactory.getLogger(Connector.class); + + private final Api api; + private final Mapper mapper; + + Connector(Api api, Mapper mapper) { + this.api = api; + this.mapper = mapper; + } + + public Message sendMessage(SendMessageRequest sendMessageRequest) { + final Message message = sendMessageRequest.getMessage(); + + try { + final SendMessagePayload sendMessagePayload = mapper.fromSendMessageRequest(sendMessageRequest); + + api.sendMessage(sendMessageRequest.getSourceConversationId(), sendMessagePayload); + + updateDeliveryState(message, DeliveryState.DELIVERED); + return message; + } catch (ApiException e) { + log.error(String.format("Failed to send a message to Google \n SendMessageRequest: %s \n Error Message: %s \n", sendMessageRequest, e.getMessage()), e); + } catch (Exception e) { + log.error(String.format("Failed to send a message to Google \n SendMessageRequest: %s", sendMessageRequest), e); + } + + updateDeliveryState(message, DeliveryState.FAILED); + return message; + } + + @Bean + public IgnoreAuthPattern ignoreAuthPattern() { + return new IgnoreAuthPattern("/google"); + } + + @Bean + public RequestLoggingIgnorePatterns requestLoggingIgnorePatterns() { + return new RequestLoggingIgnorePatterns(List.of("/google")); + } +} diff --git a/backend/sources/google/sender/src/main/java/co/airy/core/sources/google/GoogleConfig.java b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/GoogleConfig.java similarity index 99% rename from backend/sources/google/sender/src/main/java/co/airy/core/sources/google/GoogleConfig.java rename to backend/sources/google/connector/src/main/java/co/airy/core/sources/google/GoogleConfig.java index 829d476178..c1b0ea3cbe 100644 --- a/backend/sources/google/sender/src/main/java/co/airy/core/sources/google/GoogleConfig.java +++ b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/GoogleConfig.java @@ -12,7 +12,9 @@ @Configuration public class GoogleConfig { private static final Logger log = AiryLoggerFactory.getLogger(GoogleConfig.class); + final ObjectMapper objectMapper; + public GoogleConfig(ObjectMapper objectMapper) { this.objectMapper = objectMapper; } diff --git a/backend/sources/google/sender/src/main/java/co/airy/core/sources/google/Sender.java b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/Stores.java similarity index 56% rename from backend/sources/google/sender/src/main/java/co/airy/core/sources/google/Sender.java rename to backend/sources/google/connector/src/main/java/co/airy/core/sources/google/Stores.java index bdf2ef4082..c329992b68 100644 --- a/backend/sources/google/sender/src/main/java/co/airy/core/sources/google/Sender.java +++ b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/Stores.java @@ -1,45 +1,43 @@ package co.airy.core.sources.google; +import co.airy.avro.communication.Channel; import co.airy.avro.communication.DeliveryState; import co.airy.avro.communication.Message; import co.airy.avro.communication.SenderType; -import co.airy.core.sources.google.model.SendMessagePayload; import co.airy.core.sources.google.model.SendMessageRequest; -import co.airy.core.sources.google.services.Api; -import co.airy.core.sources.google.services.Mapper; +import co.airy.kafka.schema.application.ApplicationCommunicationChannels; import co.airy.kafka.schema.application.ApplicationCommunicationMessages; import co.airy.kafka.streams.KafkaStreamsWrapper; -import co.airy.log.AiryLoggerFactory; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KTable; -import org.slf4j.Logger; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; import org.springframework.beans.factory.DisposableBean; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.ApplicationListener; import org.springframework.stereotype.Component; -import static co.airy.avro.communication.MessageRepository.updateDeliveryState; - @Component -public class Sender implements DisposableBean, ApplicationListener { - private static final Logger log = AiryLoggerFactory.getLogger(Sender.class); - private static final String appId = "sources.google.Sender"; +public class Stores implements DisposableBean, ApplicationListener { + private static final String appId = "sources.google.ConnectorStores"; + private final String channelsStore = "channels-store"; + private static final String applicationCommunicationChannels = new ApplicationCommunicationChannels().name(); private final KafkaStreamsWrapper streams; - private final Api api; - private final Mapper mapper; + private final Connector connector; - Sender(KafkaStreamsWrapper streams, Api api, Mapper mapper) { + Stores(KafkaStreamsWrapper streams, Connector connector) { this.streams = streams; - this.api = api; - this.mapper = mapper; + this.connector = connector; } - public void startStream() { + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { final StreamsBuilder builder = new StreamsBuilder(); + builder.table(applicationCommunicationChannels, Materialized.as(channelsStore)); + final KStream messageStream = builder.stream(new ApplicationCommunicationMessages().name()) .filter((messageId, message) -> "google".equalsIgnoreCase(message.getSource())) .selectKey((messageId, message) -> message.getConversationId()); @@ -55,39 +53,15 @@ public void startStream() { }); messageStream.filter((messageId, message) -> DeliveryState.PENDING.equals(message.getDeliveryState())) - .join(contextTable, (message, sendMessageRequest) -> { - sendMessageRequest.setMessage(message); - return sendMessageRequest; - }) - .mapValues(this::sendMessage) + .join(contextTable, (message, sendMessageRequest) -> sendMessageRequest.toBuilder().message(message).build()) + .mapValues(connector::sendMessage) .to(new ApplicationCommunicationMessages().name()); streams.start(builder.build(), appId); } - private Message sendMessage(SendMessageRequest sendMessageRequest) { - final Message message = sendMessageRequest.getMessage(); - - try { - final SendMessagePayload sendMessagePayload = mapper.fromSendMessageRequest(sendMessageRequest); - - api.sendMessage(sendMessageRequest.getSourceConversationId(), sendMessagePayload); - - updateDeliveryState(message, DeliveryState.DELIVERED); - return message; - } catch (ApiException e) { - log.error(String.format("Failed to send a message to Google \n SendMessageRequest: %s \n Error Message: %s \n", sendMessageRequest, e.getMessage()), e); - } catch (Exception e) { - log.error(String.format("Failed to send a message to Google \n SendMessageRequest: %s", sendMessageRequest), e); - } - - updateDeliveryState(message, DeliveryState.FAILED); - return message; - } - - @Override - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - startStream(); + public ReadOnlyKeyValueStore getChannelsStore() { + return streams.acquireLocalStore(channelsStore); } @Override diff --git a/backend/sources/google/webhook/src/main/java/co/airy/core/sources/google/GoogleWebhook.java b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/WebhookController.java similarity index 73% rename from backend/sources/google/webhook/src/main/java/co/airy/core/sources/google/GoogleWebhook.java rename to backend/sources/google/connector/src/main/java/co/airy/core/sources/google/WebhookController.java index dea4af3ede..ef9e0e3623 100644 --- a/backend/sources/google/webhook/src/main/java/co/airy/core/sources/google/GoogleWebhook.java +++ b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/WebhookController.java @@ -1,8 +1,7 @@ package co.airy.core.sources.google; import co.airy.kafka.schema.source.SourceGoogleEvents; -import co.airy.payload.response.EmptyResponsePayload; -import co.airy.spring.kafka.healthcheck.ProducerHealthCheck; +import co.airy.spring.web.payload.EmptyResponsePayload; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; @@ -10,9 +9,6 @@ import org.apache.kafka.common.serialization.StringSerializer; import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.actuate.health.Health; -import org.springframework.boot.actuate.health.HealthIndicator; -import org.springframework.boot.actuate.health.Status; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PostMapping; @@ -29,19 +25,15 @@ import java.util.Properties; import java.util.UUID; -public class GoogleWebhook implements HealthIndicator, DisposableBean { - +public class WebhookController implements DisposableBean { private final String sourceGoogleEvents = new SourceGoogleEvents().name(); private final String partnerKey; private static final String HMAC_SHA512 = "HmacSHA512"; private final Producer producer; - private final ProducerHealthCheck producerHealthCheck; - GoogleWebhook(ProducerHealthCheck producerHealthCheck, - @Value("${kafka.brokers}") String brokers, - @Value("${google.partner-key}") String partnerKey) { - this.producerHealthCheck = producerHealthCheck; + WebhookController(@Value("${kafka.brokers}") String brokers, + @Value("${google.partner-key}") String partnerKey) { this.partnerKey = partnerKey; final Properties props = new Properties(); @@ -52,29 +44,18 @@ public class GoogleWebhook implements HealthIndicator, DisposableBean { props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); producer = new KafkaProducer<>(props); - } @Override - public void destroy() throws Exception { + public void destroy() { if (producer != null) { producer.close(Duration.ofSeconds(10)); } } - @Override - public Health health() { - try { - producerHealthCheck.sendHealthCheck(); - return Health.status(Status.UP).build(); - } catch (Exception e) { - return Health.down(e).build(); - } - } - @PostMapping("/google") - ResponseEntity accept(@RequestBody String event, @RequestHeader("X-Goog-Signature") String signature) throws NoSuchAlgorithmException, InvalidKeyException { - if(!validRequest(event, signature)) { + ResponseEntity accept(@RequestBody String event, @RequestHeader("X-Goog-Signature") String signature) { + if (!validRequest(event, signature)) { return ResponseEntity.status(HttpStatus.FORBIDDEN).body(new EmptyResponsePayload()); } @@ -86,7 +67,6 @@ ResponseEntity accept(@RequestBody String event, @RequestHeader("X-Goog-Signa } catch (Exception e) { return new ResponseEntity<>(HttpStatus.SERVICE_UNAVAILABLE); } - } private boolean validRequest(String payload, String signature) { diff --git a/backend/sources/google/sender/src/main/java/co/airy/core/sources/google/model/GoogleServiceAccount.java b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/model/GoogleServiceAccount.java similarity index 100% rename from backend/sources/google/sender/src/main/java/co/airy/core/sources/google/model/GoogleServiceAccount.java rename to backend/sources/google/connector/src/main/java/co/airy/core/sources/google/model/GoogleServiceAccount.java diff --git a/backend/sources/google/sender/src/main/java/co/airy/core/sources/google/model/SendMessagePayload.java b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/model/SendMessagePayload.java similarity index 100% rename from backend/sources/google/sender/src/main/java/co/airy/core/sources/google/model/SendMessagePayload.java rename to backend/sources/google/connector/src/main/java/co/airy/core/sources/google/model/SendMessagePayload.java diff --git a/backend/sources/google/sender/src/main/java/co/airy/core/sources/google/model/SendMessageRequest.java b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/model/SendMessageRequest.java similarity index 93% rename from backend/sources/google/sender/src/main/java/co/airy/core/sources/google/model/SendMessageRequest.java rename to backend/sources/google/connector/src/main/java/co/airy/core/sources/google/model/SendMessageRequest.java index 28c1defeca..4c60c81846 100644 --- a/backend/sources/google/sender/src/main/java/co/airy/core/sources/google/model/SendMessageRequest.java +++ b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/model/SendMessageRequest.java @@ -9,7 +9,7 @@ import java.io.Serializable; @Data -@Builder +@Builder(toBuilder = true) @NoArgsConstructor @AllArgsConstructor public class SendMessageRequest implements Serializable { diff --git a/backend/sources/google/sender/src/main/java/co/airy/core/sources/google/services/Api.java b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/services/Api.java similarity index 100% rename from backend/sources/google/sender/src/main/java/co/airy/core/sources/google/services/Api.java rename to backend/sources/google/connector/src/main/java/co/airy/core/sources/google/services/Api.java diff --git a/backend/sources/google/sender/src/main/java/co/airy/core/sources/google/services/Mapper.java b/backend/sources/google/connector/src/main/java/co/airy/core/sources/google/services/Mapper.java similarity index 100% rename from backend/sources/google/sender/src/main/java/co/airy/core/sources/google/services/Mapper.java rename to backend/sources/google/connector/src/main/java/co/airy/core/sources/google/services/Mapper.java diff --git a/backend/sources/google/webhook/src/main/resources/application.properties b/backend/sources/google/connector/src/main/resources/application.properties similarity index 52% rename from backend/sources/google/webhook/src/main/resources/application.properties rename to backend/sources/google/connector/src/main/resources/application.properties index 64d3fde618..c700759594 100644 --- a/backend/sources/google/webhook/src/main/resources/application.properties +++ b/backend/sources/google/connector/src/main/resources/application.properties @@ -1,3 +1,6 @@ +auth.jwt-secret=${JWT_SECRET} +google.auth.sa=${GOOGLE_SA_FILE} +google.partner-key=${GOOGLE_PARTNER_KEY} kafka.brokers=${KAFKA_BROKERS} kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} -google.partner-key=${GOOGLE_PARTNER_KEY} +kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS} diff --git a/backend/sources/google/sender/src/test/java/co/airy/core/sources/google/SendMessageTest.java b/backend/sources/google/connector/src/test/java/co/airy/core/sources/google/SendMessageTest.java similarity index 91% rename from backend/sources/google/sender/src/test/java/co/airy/core/sources/google/SendMessageTest.java rename to backend/sources/google/connector/src/test/java/co/airy/core/sources/google/SendMessageTest.java index 3a0b17b956..5ed3139880 100644 --- a/backend/sources/google/sender/src/test/java/co/airy/core/sources/google/SendMessageTest.java +++ b/backend/sources/google/connector/src/test/java/co/airy/core/sources/google/SendMessageTest.java @@ -6,6 +6,7 @@ import co.airy.core.sources.google.model.SendMessagePayload; import co.airy.core.sources.google.services.Api; import co.airy.kafka.schema.Topic; +import co.airy.kafka.schema.application.ApplicationCommunicationChannels; import co.airy.kafka.schema.application.ApplicationCommunicationMessages; import co.airy.kafka.test.KafkaTestHelper; import co.airy.kafka.test.junit.SharedKafkaTestResource; @@ -46,19 +47,24 @@ class SendMessageTest { private static KafkaTestHelper kafkaTestHelper; private static final Topic applicationCommunicationMessages = new ApplicationCommunicationMessages(); + private static final Topic applicationCommunicationChannels = new ApplicationCommunicationChannels(); @MockBean private Api api; @Autowired @InjectMocks - private Sender worker; + private Connector worker; - private static boolean streamInitialized = false; + @Autowired + private Stores stores; @BeforeAll static void beforeAll() throws Exception { - kafkaTestHelper = new KafkaTestHelper(sharedKafkaTestResource, applicationCommunicationMessages); + kafkaTestHelper = new KafkaTestHelper(sharedKafkaTestResource, + applicationCommunicationMessages, + applicationCommunicationChannels + ); kafkaTestHelper.beforeAll(); } @@ -71,10 +77,7 @@ static void afterAll() throws Exception { void beforeEach() throws InterruptedException { MockitoAnnotations.initMocks(this); - if (!streamInitialized) { - retryOnException(() -> assertEquals(worker.getStreamState(), RUNNING), "Failed to reach RUNNING state."); - streamInitialized = true; - } + retryOnException(() -> assertEquals(stores.getStreamState(), RUNNING), "Failed to reach RUNNING state."); } @Test diff --git a/backend/sources/google/sender/src/test/resources/test.properties b/backend/sources/google/connector/src/test/resources/test.properties similarity index 55% rename from backend/sources/google/sender/src/test/resources/test.properties rename to backend/sources/google/connector/src/test/resources/test.properties index 932db12288..c3d3e3d454 100644 --- a/backend/sources/google/sender/src/test/resources/test.properties +++ b/backend/sources/google/connector/src/test/resources/test.properties @@ -1 +1,4 @@ google.auth.sa={"type":"service_account","project_id":"airy","private_key_id":"no","private_key":"nokey","client_email":"no","client_id":"no","auth_uri":"no","token_uri":"no","no":"no","client_x509_cert_url":"no"} +google.partner-key=whatever +auth.jwt-secret=42424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242 +kafka.commit-interval-ms=100 diff --git a/backend/sources/google/events-router/BUILD b/backend/sources/google/events-router/BUILD index 04522d102e..67c348347e 100644 --- a/backend/sources/google/events-router/BUILD +++ b/backend/sources/google/events-router/BUILD @@ -4,10 +4,10 @@ load("//tools/build:container_push.bzl", "container_push") app_deps = [ "//backend:base_app", - "//backend:channel", - "//backend:message", + "//backend/model/channel:channel", + "//backend/model/message:message", + "//backend/model/metadata:metadata", "//lib/java/uuid", - "//lib/java/payload", "//lib/java/kafka/schema:source-google-events", "//lib/java/spring/kafka/core:spring-kafka-core", "//lib/java/spring/kafka/streams:spring-kafka-streams", diff --git a/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/GoogleEventInfo.java b/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/EventInfo.java similarity index 77% rename from backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/GoogleEventInfo.java rename to backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/EventInfo.java index ff4ccda28c..894379ac13 100644 --- a/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/GoogleEventInfo.java +++ b/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/EventInfo.java @@ -12,10 +12,10 @@ @AllArgsConstructor @NoArgsConstructor @Builder(toBuilder = true) -public class GoogleEventInfo implements Serializable { +public class EventInfo implements Serializable { private String agentId; - private String conversationId; - private String eventPayload; + private String sourceConversationId; + private WebhookEvent event; private Channel channel; private Long timestamp; private boolean isMessage; diff --git a/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/EventsRouter.java b/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/EventsRouter.java index 555499cce8..fc5e4f9e73 100644 --- a/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/EventsRouter.java +++ b/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/EventsRouter.java @@ -4,14 +4,17 @@ import co.airy.avro.communication.ChannelConnectionState; import co.airy.avro.communication.DeliveryState; import co.airy.avro.communication.Message; +import co.airy.avro.communication.Metadata; import co.airy.avro.communication.SenderType; import co.airy.kafka.schema.application.ApplicationCommunicationChannels; import co.airy.kafka.schema.application.ApplicationCommunicationMessages; +import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; import co.airy.kafka.schema.source.SourceGoogleEvents; import co.airy.kafka.streams.KafkaStreamsWrapper; import co.airy.log.AiryLoggerFactory; import co.airy.uuid.UUIDv5; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.avro.specific.SpecificRecordBase; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsBuilder; @@ -24,8 +27,13 @@ import org.springframework.stereotype.Component; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; import java.util.Map; +import static co.airy.model.metadata.MetadataRepository.getId; +import static co.airy.core.sources.google.InfoExtractor.getMetadataFromContext; + @Component public class EventsRouter implements DisposableBean, ApplicationListener { private static final String appId = "sources.google.EventsRouter"; @@ -40,7 +48,7 @@ public EventsRouter(KafkaStreamsWrapper streams, @Qualifier("googleObjectMapper" } @Override - public void destroy() throws Exception { + public void destroy() { if (streams != null) { streams.close(); } @@ -53,6 +61,8 @@ public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { private void startStream() { final StreamsBuilder builder = new StreamsBuilder(); + final String applicationCommunicationMessages = new ApplicationCommunicationMessages().name(); + final String applicationCommunicationMetadata = new ApplicationCommunicationMetadata().name(); // Channels table KTable channelsTable = builder.stream(new ApplicationCommunicationChannels().name()) @@ -75,51 +85,78 @@ private void startStream() { return KeyValue.pair("skip", null); } - GoogleEventInfo googleEventInfo = GoogleInfoExtractor.extract(webhookEvent); - googleEventInfo.setEventPayload(sourceEvent); - googleEventInfo.setTimestamp(Instant.parse(webhookEvent.getSendTime()).toEpochMilli()); + final EventInfo eventInfo = InfoExtractor.extract(webhookEvent); + eventInfo.setEvent(webhookEvent); + eventInfo.setTimestamp(Instant.parse(webhookEvent.getSendTime()).toEpochMilli()); - if (!webhookEvent.isMessage()) { - return KeyValue.pair(googleEventInfo.getAgentId(), null); + if (!webhookEvent.hasMessage() && !webhookEvent.hasContext()) { + return KeyValue.pair(eventInfo.getAgentId(), null); } - return KeyValue.pair(googleEventInfo.getAgentId(), googleEventInfo); + return KeyValue.pair(eventInfo.getAgentId(), eventInfo); }) .filter((agentId, event) -> event != null) .join(channelsTable, (event, channel) -> event.toBuilder().channel(channel).build()) - .map((agentId, event) -> { + .flatMap((agentId, event) -> { final Channel channel = event.getChannel(); - final String payload = event.getEventPayload(); - - final String messageId = UUIDv5.fromNamespaceAndName(channel.getId(), payload).toString(); - final String conversationId = UUIDv5.fromNamespaceAndName(channel.getId(), event.getConversationId()).toString(); - final String sourceConversationId = event.getConversationId(); - - Message.Builder messageBuilder = Message.newBuilder(); - return KeyValue.pair( - messageId, - messageBuilder - .setSource(channel.getSource()) - .setDeliveryState(DeliveryState.DELIVERED) - .setId(messageId) - .setChannelId(channel.getId()) - .setConversationId(conversationId) - .setSenderType(SenderType.SOURCE_CONTACT) - .setContent(payload) - .setSenderId(sourceConversationId) - .setHeaders(Map.of()) // TODO we can add place Id - .setSentAt(event.getTimestamp()) - .setUpdatedAt(null) - .build() - ); + final WebhookEvent webhookEvent = event.getEvent(); + + String payload; + try { + payload = objectMapper.writeValueAsString(webhookEvent); + } catch (Exception e) { + throw new RuntimeException(e); + } + + final String sourceConversationId = event.getSourceConversationId(); + final String conversationId = UUIDv5.fromNamespaceAndName(channel.getId(), sourceConversationId).toString(); + final List> records = new ArrayList<>(); + + if (webhookEvent.hasMessage()) { + final String messageId = UUIDv5.fromNamespaceAndName(channel.getId(), payload).toString(); + records.add(KeyValue.pair(messageId, + Message.newBuilder() + .setSource(channel.getSource()) + .setDeliveryState(DeliveryState.DELIVERED) + .setId(messageId) + .setChannelId(channel.getId()) + .setConversationId(conversationId) + .setSenderType(SenderType.SOURCE_CONTACT) + .setContent(payload) + .setSenderId(sourceConversationId) + .setHeaders(Map.of()) + .setSentAt(event.getTimestamp()) + .setUpdatedAt(null) + .build() + )); + } + + if (webhookEvent.hasContext()) { + final List metadataFromContext = getMetadataFromContext(conversationId, webhookEvent); + + for (Metadata metadata : metadataFromContext) { + records.add(KeyValue.pair(getId(metadata).toString(), metadata)); + } + } + + return records; }) - .filter((messageId, message) -> message != null) - .to(new ApplicationCommunicationMessages().name()); + .filter((recordId, record) -> record != null) + .to((recordId, record, context) -> { + if (record instanceof Metadata) { + return applicationCommunicationMetadata; + } + if (record instanceof Message) { + return applicationCommunicationMessages; + } + + throw new IllegalStateException("Unknown type for record " + record); + }); streams.start(builder.build(), appId); } - // visible for testing + // Visible for testing KafkaStreams.State getStreamState() { return streams.state(); } diff --git a/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/GoogleInfoExtractor.java b/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/GoogleInfoExtractor.java deleted file mode 100644 index ed7e626f25..0000000000 --- a/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/GoogleInfoExtractor.java +++ /dev/null @@ -1,30 +0,0 @@ -package co.airy.core.sources.google; - -import co.airy.log.AiryLoggerFactory; -import org.slf4j.Logger; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class GoogleInfoExtractor { - private static final Logger log = AiryLoggerFactory.getLogger(GoogleInfoExtractor.class); - - private static final Pattern agentPattern = Pattern.compile("brands/(.*?)/agents/(.*)"); - - static GoogleEventInfo extract(WebhookEvent event) { - try { - Matcher agentBrandMatcher = agentPattern.matcher(event.getAgent()); - agentBrandMatcher.find(); - final String agentId = agentBrandMatcher.group(2); - - return GoogleEventInfo.builder() - .agentId(agentId) - .conversationId(event.getConversationId()) - .build(); - } catch (Throwable e) { - log.info("Event {} is not parseable", event); - throw new IllegalArgumentException("Could not extract event", e); - } - } - -} diff --git a/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/InfoExtractor.java b/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/InfoExtractor.java new file mode 100644 index 0000000000..e1ddab2987 --- /dev/null +++ b/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/InfoExtractor.java @@ -0,0 +1,61 @@ +package co.airy.core.sources.google; + +import co.airy.avro.communication.Metadata; +import co.airy.model.metadata.MetadataKeys; +import co.airy.log.AiryLoggerFactory; +import com.fasterxml.jackson.databind.JsonNode; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static co.airy.model.metadata.MetadataRepository.newConversationMetadata; + +public class InfoExtractor { + private static final Logger log = AiryLoggerFactory.getLogger(InfoExtractor.class); + + private static final Pattern agentPattern = Pattern.compile("brands/(.*?)/agents/(.*)"); + + static EventInfo extract(WebhookEvent event) { + try { + Matcher agentBrandMatcher = agentPattern.matcher(event.getAgent()); + agentBrandMatcher.find(); + final String agentId = agentBrandMatcher.group(2); + + return EventInfo.builder() + .agentId(agentId) + .sourceConversationId(event.getConversationId()) + .build(); + } catch (Throwable e) { + log.info("Event {} is not parseable", event); + throw new IllegalArgumentException("Could not extract event", e); + } + } + + static List getMetadataFromContext(String conversationId, WebhookEvent webhookEvent) { + final JsonNode context = webhookEvent.getContext(); + + List metadata = new ArrayList<>(); + + final JsonNode userInfo = context.get("userInfo"); + if (userInfo != null && userInfo.has("displayName")) { + final String displayName = userInfo.get("displayName").textValue(); + final int lastIndexOf = displayName.indexOf(" "); + + // There's only a first name + if (lastIndexOf != -1) { + final String firstName = displayName.substring(0, lastIndexOf); + final String lastName = displayName.substring(lastIndexOf + 1); + + metadata.add(newConversationMetadata(conversationId, MetadataKeys.Source.Contact.FIRST_NAME, firstName)); + metadata.add(newConversationMetadata(conversationId, MetadataKeys.Source.Contact.LAST_NAME, lastName)); + } else { + metadata.add(newConversationMetadata(conversationId, MetadataKeys.Source.Contact.FIRST_NAME, displayName)); + } + } + + return metadata; + } +} diff --git a/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/ObjectMapperConfig.java b/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/ObjectMapperConfig.java index d8bb99af3e..adc68f94e9 100644 --- a/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/ObjectMapperConfig.java +++ b/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/ObjectMapperConfig.java @@ -1,5 +1,6 @@ package co.airy.core.sources.google; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; @@ -15,6 +16,7 @@ public ObjectMapper objectMapper() { return new ObjectMapper() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) .configure(DeserializationFeature.FAIL_ON_MISSING_EXTERNAL_TYPE_ID_PROPERTY, false) + .setSerializationInclusion(JsonInclude.Include.NON_NULL) .setPropertyNamingStrategy(PropertyNamingStrategy.LOWER_CAMEL_CASE); } } diff --git a/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/WebhookEvent.java b/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/WebhookEvent.java index bf502e93a8..40abcba871 100644 --- a/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/WebhookEvent.java +++ b/backend/sources/google/events-router/src/main/java/co/airy/core/sources/google/WebhookEvent.java @@ -1,5 +1,6 @@ package co.airy.core.sources.google; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import lombok.NoArgsConstructor; @@ -25,6 +26,7 @@ public class WebhookEvent { private String sendTime; + @JsonIgnore public JsonNode getPayload() { return Stream.of(this.message, this.suggestionResponse, this.surveyResponse, this.receipts, this.userStatus) .filter(Objects::nonNull) @@ -32,7 +34,13 @@ public JsonNode getPayload() { .get(); } - public boolean isMessage() { + @JsonIgnore + public boolean hasMessage() { return this.message != null; } + + @JsonIgnore + public boolean hasContext() { + return this.context != null && !this.context.isEmpty(); + } } diff --git a/backend/sources/google/events-router/src/main/resources/application.properties b/backend/sources/google/events-router/src/main/resources/application.properties index cfeec4b55c..3c1d68e8b2 100644 --- a/backend/sources/google/events-router/src/main/resources/application.properties +++ b/backend/sources/google/events-router/src/main/resources/application.properties @@ -1,2 +1,3 @@ kafka.brokers=${KAFKA_BROKERS} kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} +kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS} diff --git a/backend/sources/google/events-router/src/test/java/co/airy/core/sources/google/EventsRouterTest.java b/backend/sources/google/events-router/src/test/java/co/airy/core/sources/google/EventsRouterTest.java index daecb03cc6..04b303838c 100644 --- a/backend/sources/google/events-router/src/test/java/co/airy/core/sources/google/EventsRouterTest.java +++ b/backend/sources/google/events-router/src/test/java/co/airy/core/sources/google/EventsRouterTest.java @@ -3,9 +3,12 @@ import co.airy.avro.communication.Channel; import co.airy.avro.communication.ChannelConnectionState; import co.airy.avro.communication.Message; +import co.airy.avro.communication.Metadata; +import co.airy.model.metadata.MetadataKeys; import co.airy.kafka.schema.Topic; import co.airy.kafka.schema.application.ApplicationCommunicationChannels; import co.airy.kafka.schema.application.ApplicationCommunicationMessages; +import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; import co.airy.kafka.schema.source.SourceGoogleEvents; import co.airy.kafka.test.KafkaTestHelper; import co.airy.kafka.test.junit.SharedKafkaTestResource; @@ -19,6 +22,7 @@ import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import java.util.List; @@ -30,11 +34,10 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.collection.IsCollectionWithSize.hasSize; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -@SpringBootTest(properties = { - "kafka.cleanup=true", - "kafka.commit-interval-ms=100", -}, classes = AirySpringBootApplication.class) +@SpringBootTest(classes = AirySpringBootApplication.class) +@TestPropertySource(value = "classpath:test.properties") @ExtendWith(SpringExtension.class) public class EventsRouterTest { @RegisterExtension @@ -43,6 +46,7 @@ public class EventsRouterTest { private static final Topic sourceGoogleEvents = new SourceGoogleEvents(); private static final Topic applicationCommunicationChannels = new ApplicationCommunicationChannels(); private static final Topic applicationCommunicationMessages = new ApplicationCommunicationMessages(); + private static final Topic applicationCommunicationMetadata = new ApplicationCommunicationMetadata(); @Autowired private EventsRouter worker; @@ -52,7 +56,8 @@ static void beforeAll() throws Exception { kafkaTestHelper = new KafkaTestHelper(sharedKafkaTestResource, sourceGoogleEvents, applicationCommunicationChannels, - applicationCommunicationMessages + applicationCommunicationMessages, + applicationCommunicationMetadata ); kafkaTestHelper.beforeAll(); @@ -71,11 +76,11 @@ void beforeEach() throws InterruptedException { @Test void canRouteGoogleMessages() throws Exception { String channelId = UUID.randomUUID().toString(); - String pageId = UUID.randomUUID().toString(); + String agentId = UUID.randomUUID().toString(); kafkaTestHelper.produceRecord(new ProducerRecord<>(applicationCommunicationChannels.name(), channelId, Channel.newBuilder() .setId(channelId) .setConnectionState(ChannelConnectionState.CONNECTED) - .setSourceChannelId(pageId) + .setSourceChannelId(agentId) .setName("Awesome place") .setSource("google") .setToken("") @@ -86,10 +91,61 @@ void canRouteGoogleMessages() throws Exception { final String eventPayload = "{ \"agent\": \"brands/somebrand/agents/%s\", \"conversationId\": \"CONVERSATION_ID\", \"customAgentId\": \"CUSTOM_AGENT_ID\", \"message\": { \"messageId\": \"MESSAGE_ID\", \"name\": \"conversations/CONVERSATION_ID/messages/MESSAGE_ID\", \"text\": \"MESSAGE_TEXT\", \"createTime\": \"MESSAGE_CREATE_TIME\" }, \"context\": { \"placeId\": \"LOCATION_PLACE_ID\" }, \"sendTime\": \"2014-10-02T15:01:23.045123456Z\" }"; - List> events = List.of(new ProducerRecord<>(sourceGoogleEvents.name(), UUID.randomUUID().toString(), String.format(eventPayload, pageId))); + List> events = List.of(new ProducerRecord<>(sourceGoogleEvents.name(), UUID.randomUUID().toString(), String.format(eventPayload, agentId))); kafkaTestHelper.produceRecords(events); List messages = kafkaTestHelper.consumeValues(1, applicationCommunicationMessages.name()); assertThat(messages, hasSize(1)); } + + @Test + void canRouteGoogleMetadata() throws Exception { + String channelId = UUID.randomUUID().toString(); + String agentId = UUID.randomUUID().toString(); + kafkaTestHelper.produceRecord(new ProducerRecord<>(applicationCommunicationChannels.name(), channelId, Channel.newBuilder() + .setId(channelId) + .setConnectionState(ChannelConnectionState.CONNECTED) + .setSourceChannelId(agentId) + .setName("Awesome place") + .setSource("google") + .setToken("") + .build())); + + // Wait for the channels table to catch up + TimeUnit.SECONDS.sleep(5); + + final String displayName = "Grace Brewster Murray Hopper"; + final String singleName = "hal9000"; + + // Two different event types that both carry context + final String messagePayload = "{ \"agent\": \"brands/somebrand/agents/%s\", \"conversationId\": \"CONVERSATION_ID\"," + + " \"customAgentId\": \"CUSTOM_AGENT_ID\", \"message\": { \"messageId\": \"MESSAGE_ID\", \"name\": \"conversations/CONVERSATION_ID/messages/MESSAGE_ID\", \"text\": \"MESSAGE_TEXT\", \"createTime\": \"MESSAGE_CREATE_TIME\" }," + + " \"context\": { \"userInfo\": { \"displayName\": \"%s\" } }, \"sendTime\": \"2014-10-02T15:01:23.045123456Z\" }"; + final String userStatusPayload = "{ \"agent\": \"brands/somebrand/agents/%s\", \"conversationId\": \"CONVERSATION_ID\", " + + "\"customAgentId\": \"CUSTOM_AGENT_ID\", \"userStatus\": { \"isTyping\": true }, " + + "\"context\": { \"userInfo\": { \"displayName\": \"%s\" } }, \"sendTime\": \"2014-10-02T15:01:23.045123456Z\" }"; + + List> events = List.of( + new ProducerRecord<>(sourceGoogleEvents.name(), UUID.randomUUID().toString(), String.format(messagePayload, agentId, displayName)), + new ProducerRecord<>(sourceGoogleEvents.name(), UUID.randomUUID().toString(), String.format(userStatusPayload, agentId, singleName)) + ); + + kafkaTestHelper.produceRecords(events); + + List metadataList = kafkaTestHelper.consumeValues(3, applicationCommunicationMetadata.name()); + assertThat(metadataList, hasSize(3)); + + assertTrue(metadataList.stream().anyMatch((metadata -> + metadata.getKey().equals(MetadataKeys.Source.Contact.FIRST_NAME) && + metadata.getValue().equals(singleName) + ))); + assertTrue(metadataList.stream().anyMatch((metadata -> + metadata.getKey().equals(MetadataKeys.Source.Contact.FIRST_NAME) && + metadata.getValue().equals("Grace") + ))); + assertTrue(metadataList.stream().anyMatch((metadata -> + metadata.getKey().equals(MetadataKeys.Source.Contact.LAST_NAME) && + metadata.getValue().equals("Brewster Murray Hopper") + ))); + } } diff --git a/backend/sources/google/events-router/src/test/resources/test.properties b/backend/sources/google/events-router/src/test/resources/test.properties new file mode 100644 index 0000000000..1d10a69841 --- /dev/null +++ b/backend/sources/google/events-router/src/test/resources/test.properties @@ -0,0 +1,2 @@ +kafka.cleanup=true +kafka.commit-interval-ms=100 diff --git a/backend/sources/google/sender/src/main/resources/application.properties b/backend/sources/google/sender/src/main/resources/application.properties deleted file mode 100644 index 2736a74dd0..0000000000 --- a/backend/sources/google/sender/src/main/resources/application.properties +++ /dev/null @@ -1,3 +0,0 @@ -kafka.brokers=${KAFKA_BROKERS} -kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} -google.auth.sa=${GOOGLE_SA_FILE} diff --git a/backend/sources/google/webhook/BUILD b/backend/sources/google/webhook/BUILD deleted file mode 100644 index ac88c3e14d..0000000000 --- a/backend/sources/google/webhook/BUILD +++ /dev/null @@ -1,37 +0,0 @@ -load("//tools/build:springboot.bzl", "springboot") -load("//tools/build:junit5.bzl", "junit5") -load("//tools/build:container_push.bzl", "container_push") - -app_deps = [ - "//backend:base_app", - "//:springboot_actuator", - "//lib/java/payload", - "//lib/java/kafka/schema:source-google-events", - "//lib/java/spring/kafka/core:spring-kafka-core", - "//lib/java/spring/kafka/healthcheck", -] - -springboot( - name = "google-webhook", - srcs = glob(["src/main/java/**/*.java"]), - main_class = "co.airy.spring.core.AirySpringBootApplication", - deps = app_deps, -) - -[ - junit5( - size = "small", - file = file, - resources = glob(["src/test/resources/**/*"]), - deps = [ - ":app", - "//lib/java/kafka/test:kafka-test", - ] + app_deps, - ) - for file in glob(["src/test/java/**/*Test.java"]) -] - -container_push( - registry = "ghcr.io/airyhq/sources", - repository = "google-webhook", -) diff --git a/backend/sources/twilio/sender/BUILD b/backend/sources/twilio/connector/BUILD similarity index 72% rename from backend/sources/twilio/sender/BUILD rename to backend/sources/twilio/connector/BUILD index d9d1255d28..e02d26ce15 100644 --- a/backend/sources/twilio/sender/BUILD +++ b/backend/sources/twilio/connector/BUILD @@ -4,17 +4,21 @@ load("//tools/build:container_push.bzl", "container_push") app_deps = [ "//backend:base_app", - "//backend:channel", - "//backend:message", + "//backend/model/channel:channel", + "//backend/model/message:message", "//lib/java/log", "//lib/java/mapping", + "//lib/java/uuid", "@maven//:com_twilio_sdk_twilio", "//lib/java/spring/kafka/core:spring-kafka-core", "//lib/java/spring/kafka/streams:spring-kafka-streams", + "//lib/java/kafka/schema:source-twilio-events", + "//lib/java/spring/web:spring-web", + "//lib/java/spring/auth:spring-auth", ] springboot( - name = "sender", + name = "connector", srcs = glob(["src/main/java/**/*.java"]), main_class = "co.airy.spring.core.AirySpringBootApplication", deps = app_deps, @@ -28,6 +32,7 @@ springboot( deps = [ ":app", "//backend:base_test", + "@maven//:javax_xml_bind_jaxb_api", "//lib/java/kafka/test:kafka-test", ] + app_deps, ) @@ -36,5 +41,5 @@ springboot( container_push( registry = "ghcr.io/airyhq/sources", - repository = "twilio-sender", + repository = "twilio-connector", ) diff --git a/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/ChannelsController.java b/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/ChannelsController.java new file mode 100644 index 0000000000..58cb944c6a --- /dev/null +++ b/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/ChannelsController.java @@ -0,0 +1,135 @@ +package co.airy.core.sources.twilio; + +import co.airy.avro.communication.Channel; +import co.airy.avro.communication.ChannelConnectionState; +import co.airy.kafka.schema.application.ApplicationCommunicationChannels; +import co.airy.spring.web.payload.EmptyResponsePayload; +import co.airy.uuid.UUIDv5; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import javax.validation.Valid; +import javax.validation.constraints.NotNull; +import java.util.UUID; + +import static co.airy.model.channel.ChannelPayload.fromChannel; + +@RestController +public class ChannelsController { + private static final String applicationCommunicationChannels = new ApplicationCommunicationChannels().name(); + + private final Stores stores; + private final KafkaProducer producer; + + public ChannelsController(Stores stores, KafkaProducer producer) { + this.stores = stores; + this.producer = producer; + } + + @PostMapping("/twilio.sms.connect") + ResponseEntity connectSms(@RequestBody @Valid ConnectChannelRequestPayload requestPayload) { + final String channelId = UUIDv5.fromNamespaceAndName("twilio.sms", requestPayload.getPhoneNumber()).toString(); + + final Channel channel = Channel.newBuilder() + .setId(channelId) + .setConnectionState(ChannelConnectionState.CONNECTED) + .setSource("twilio.sms") + .setSourceChannelId(requestPayload.getPhoneNumber()) + .setName(requestPayload.getName()) + .setImageUrl(requestPayload.getImageUrl()) + .build(); + + return connectChannel(channel); + } + + @PostMapping("/twilio.whatsapp.connect") + ResponseEntity connectWhatsapp(@RequestBody @Valid ConnectChannelRequestPayload requestPayload) { + final String phoneNumber = "whatsapp:" + requestPayload.getPhoneNumber(); + final String channelId = UUIDv5.fromNamespaceAndName("twilio.whatsapp", phoneNumber).toString(); + + final Channel channel = Channel.newBuilder() + .setId(channelId) + .setConnectionState(ChannelConnectionState.CONNECTED) + .setSource("twilio.whatsapp") + .setSourceChannelId(phoneNumber) + .setName(requestPayload.getName()) + .setImageUrl(requestPayload.getImageUrl()) + .build(); + + return connectChannel(channel); + } + + private ResponseEntity connectChannel(Channel channel) { + try { + producer.send(new ProducerRecord<>(applicationCommunicationChannels, channel.getId(), channel)).get(); + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build(); + } + + return ResponseEntity.ok(fromChannel(channel)); + } + + @PostMapping("/twilio.sms.disconnect") + ResponseEntity disconnectSms(@RequestBody @Valid DisconnectChannelRequestPayload requestPayload) { + return disconnect(requestPayload); + } + + @PostMapping("/twilio.whatsapp.disconnect") + ResponseEntity disconnectWhatsapp(@RequestBody @Valid DisconnectChannelRequestPayload requestPayload) { + return disconnect(requestPayload); + } + + private ResponseEntity disconnect(@RequestBody @Valid DisconnectChannelRequestPayload requestPayload) { + final String channelId = requestPayload.getChannelId().toString(); + + final Channel channel = stores.getChannelsStore().get(channelId); + + if (channel == null) { + return ResponseEntity.notFound().build(); + } + + if (channel.getConnectionState().equals(ChannelConnectionState.DISCONNECTED)) { + return ResponseEntity.accepted().build(); + } + + channel.setConnectionState(ChannelConnectionState.DISCONNECTED); + channel.setToken(null); + + try { + producer.send(new ProducerRecord<>(applicationCommunicationChannels, channel.getId(), channel)).get(); + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build(); + } + + return ResponseEntity.ok(new EmptyResponsePayload()); + } +} + +@Data +@NoArgsConstructor +@AllArgsConstructor +class ConnectChannelRequestPayload { + @NotNull + private String phoneNumber; + + @NotNull + private String name; + + private String imageUrl; +} + +@Data +@NoArgsConstructor +@AllArgsConstructor +class DisconnectChannelRequestPayload { + @NotNull + private UUID channelId; +} diff --git a/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/Connector.java b/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/Connector.java new file mode 100644 index 0000000000..69107ca5ff --- /dev/null +++ b/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/Connector.java @@ -0,0 +1,69 @@ +package co.airy.core.sources.twilio; + +import co.airy.avro.communication.DeliveryState; +import co.airy.avro.communication.Message; +import co.airy.core.sources.twilio.dto.SendMessageRequest; +import co.airy.core.sources.twilio.services.Api; +import co.airy.log.AiryLoggerFactory; +import co.airy.mapping.ContentMapper; +import co.airy.mapping.model.Text; +import co.airy.spring.auth.IgnoreAuthPattern; +import co.airy.spring.web.filters.RequestLoggingIgnorePatterns; +import com.twilio.exception.ApiException; +import org.slf4j.Logger; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +import java.util.List; + +import static co.airy.model.message.MessageRepository.updateDeliveryState; + +@Component +public class Connector { + private static final Logger log = AiryLoggerFactory.getLogger(Connector.class); + + private final Api api; + private final ContentMapper mapper; + + Connector(Api api, ContentMapper mapper) { + this.api = api; + this.mapper = mapper; + } + + public Message sendMessage(SendMessageRequest sendMessageRequest) { + final Message message = sendMessageRequest.getMessage(); + final String from = sendMessageRequest.getChannel().getSourceChannelId(); + final String to = sendMessageRequest.getSourceConversationId(); + try { + // TODO Figure out how we can let clients know which outbound message types are supported + final Text text = (Text) mapper.render(message) + .stream() + .filter(c -> c instanceof Text) + .findFirst() + .orElse(null); + + api.sendMessage(from, to, text.getText()); + + updateDeliveryState(message, DeliveryState.DELIVERED); + return message; + } catch (ApiException e) { + log.error(String.format("Failed to send a message to Twilio \n SendMessageRequest: %s \n Error Message: %s \n", sendMessageRequest, e.getMessage()), e); + } catch (Exception e) { + log.error(String.format("Failed to send a message to Twilio \n SendMessageRequest: %s", sendMessageRequest), e); + } + + updateDeliveryState(message, DeliveryState.FAILED); + return message; + } + + @Bean + public IgnoreAuthPattern ignoreAuthPattern() { + return new IgnoreAuthPattern("/twilio"); + } + + @Bean + public RequestLoggingIgnorePatterns requestLoggingIgnorePatterns() { + return new RequestLoggingIgnorePatterns(List.of("/twilio")); + } + +} diff --git a/backend/sources/twilio/sender/src/main/java/co/airy/core/sources/twilio/Sender.java b/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/Stores.java similarity index 50% rename from backend/sources/twilio/sender/src/main/java/co/airy/core/sources/twilio/Sender.java rename to backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/Stores.java index 3abff6f25d..afcb3b0d1f 100644 --- a/backend/sources/twilio/sender/src/main/java/co/airy/core/sources/twilio/Sender.java +++ b/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/Stores.java @@ -5,49 +5,49 @@ import co.airy.avro.communication.DeliveryState; import co.airy.avro.communication.Message; import co.airy.avro.communication.SenderType; -import co.airy.core.sources.twilio.model.SendMessageRequest; -import co.airy.core.sources.twilio.services.Api; +import co.airy.core.sources.twilio.dto.SendMessageRequest; import co.airy.kafka.schema.application.ApplicationCommunicationChannels; import co.airy.kafka.schema.application.ApplicationCommunicationMessages; import co.airy.kafka.streams.KafkaStreamsWrapper; -import co.airy.log.AiryLoggerFactory; -import co.airy.mapping.ContentMapper; -import co.airy.mapping.model.Text; -import com.twilio.exception.ApiException; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KTable; -import org.slf4j.Logger; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; import org.springframework.beans.factory.DisposableBean; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.ApplicationListener; import org.springframework.stereotype.Component; -import static co.airy.avro.communication.MessageRepository.updateDeliveryState; - @Component -public class Sender implements DisposableBean, ApplicationListener { - private static final Logger log = AiryLoggerFactory.getLogger(Sender.class); - private static final String appId = "sources.twilio.Sender"; +public class Stores implements DisposableBean, ApplicationListener { + + private static final String applicationCommunicationChannels = new ApplicationCommunicationChannels().name(); + + private static final String appId = "sources.twilio.ConnectorStores"; + private final String channelsStore = "channels-store"; private final KafkaStreamsWrapper streams; - private final Api api; - private final ContentMapper mapper; + private final Connector connector; - Sender(KafkaStreamsWrapper streams, Api api, ContentMapper mapper) { + Stores(KafkaStreamsWrapper streams, Connector connector) { this.streams = streams; - this.api = api; - this.mapper = mapper; + this.connector = connector; } - public void startStream() { + @Override + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { final StreamsBuilder builder = new StreamsBuilder(); + KStream channelStream = builder.stream(applicationCommunicationChannels); + // Channels table - KTable channelsTable = builder.table(new ApplicationCommunicationChannels().name()) + KTable channelsTable = channelStream .filter((sourceChannelId, channel) -> channel.getSource().startsWith("twilio") - && channel.getConnectionState().equals(ChannelConnectionState.CONNECTED)); + && channel.getConnectionState().equals(ChannelConnectionState.CONNECTED)).toTable(); + + channelStream.toTable(Materialized.as(channelsStore)); final KStream messageStream = builder.stream(new ApplicationCommunicationMessages().name()) .filter((messageId, message) -> message.getSource().startsWith("twilio")) @@ -57,55 +57,28 @@ public void startStream() { .groupByKey() .aggregate(SendMessageRequest::new, (conversationId, message, aggregate) -> { + SendMessageRequest.SendMessageRequestBuilder sendMessageRequestBuilder = aggregate.toBuilder(); if (SenderType.SOURCE_CONTACT.equals(message.getSenderType())) { - aggregate.setSourceConversationId(message.getSenderId()); + sendMessageRequestBuilder.sourceConversationId(message.getSenderId()); } - aggregate.setChannelId(message.getChannelId()); + sendMessageRequestBuilder.channelId(message.getChannelId()); - return aggregate; + return sendMessageRequestBuilder.build(); }) - .join(channelsTable, SendMessageRequest::getChannelId, (aggregate, channel) -> { - aggregate.setChannel(channel); - return aggregate; - }); + .join(channelsTable, SendMessageRequest::getChannelId, + (aggregate, channel) -> aggregate.toBuilder().channel(channel).build()); messageStream.filter((messageId, message) -> DeliveryState.PENDING.equals(message.getDeliveryState())) - .join(contextTable, (message, sendMessageRequest) -> { - sendMessageRequest.setMessage(message); - return sendMessageRequest; - }) - .mapValues(this::sendMessage) + .join(contextTable, (message, sendMessageRequest) -> sendMessageRequest.toBuilder().message(message).build()) + .mapValues(connector::sendMessage) .to(new ApplicationCommunicationMessages().name()); streams.start(builder.build(), appId); } - private Message sendMessage(SendMessageRequest sendMessageRequest) { - final Message message = sendMessageRequest.getMessage(); - final String from = sendMessageRequest.getChannel().getSourceChannelId(); - final String to = sendMessageRequest.getSourceConversationId(); - - try { - // TODO Figure out how we can let clients know which outbound message types are supported - final Text content = (Text) mapper.render(message); - api.sendMessage(from, to, content.getText()); - - updateDeliveryState(message, DeliveryState.DELIVERED); - return message; - } catch (ApiException e) { - log.error(String.format("Failed to send a message to Twilio \n SendMessageRequest: %s \n Error Message: %s \n", sendMessageRequest, e.getMessage()), e); - } catch (Exception e) { - log.error(String.format("Failed to send a message to Twilio \n SendMessageRequest: %s", sendMessageRequest), e); - } - - updateDeliveryState(message, DeliveryState.FAILED); - return message; - } - - @Override - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - startStream(); + public ReadOnlyKeyValueStore getChannelsStore() { + return streams.acquireLocalStore(channelsStore); } @Override diff --git a/backend/sources/twilio/webhook/src/main/java/co/airy/core/sources/twilio/TwilioWebhook.java b/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/WebhookController.java similarity index 80% rename from backend/sources/twilio/webhook/src/main/java/co/airy/core/sources/twilio/TwilioWebhook.java rename to backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/WebhookController.java index 7d2f03bd2e..41d01c8233 100644 --- a/backend/sources/twilio/webhook/src/main/java/co/airy/core/sources/twilio/TwilioWebhook.java +++ b/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/WebhookController.java @@ -1,7 +1,6 @@ -package co.airy.core.twilio; +package co.airy.core.sources.twilio; import co.airy.kafka.schema.source.SourceTwilioEvents; -import co.airy.spring.kafka.healthcheck.ProducerHealthCheck; import com.twilio.security.RequestValidator; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; @@ -10,8 +9,6 @@ import org.apache.kafka.common.serialization.StringSerializer; import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.actuate.health.Health; -import org.springframework.boot.actuate.health.HealthIndicator; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -33,18 +30,14 @@ import static java.util.stream.Collectors.toMap; @RestController -public class TwilioWebhook implements HealthIndicator, DisposableBean { - +public class WebhookController implements DisposableBean { private final String sourceTwilioEvents = new SourceTwilioEvents().name(); private final Producer producer; private final String authToken; - private final ProducerHealthCheck producerHealthCheck; - public TwilioWebhook(ProducerHealthCheck producerHealthCheck, - @Value("${twilio.token}") String authToken, - @Value("${kafka.brokers}") String brokers) { - this.producerHealthCheck = producerHealthCheck; + public WebhookController(@Value("${twilio.auth-token}") String authToken, + @Value("${kafka.brokers}") String brokers) { this.authToken = authToken; final Properties props = new Properties(); @@ -57,15 +50,6 @@ public TwilioWebhook(ProducerHealthCheck producerHealthCheck, } - public Health health() { - try { - producerHealthCheck.sendHealthCheck(); - return Health.up().build(); - } catch (Exception e) { - return Health.down(e).build(); - } - } - @PostMapping( path = "/twilio", consumes = {MediaType.APPLICATION_FORM_URLENCODED_VALUE}) @@ -103,9 +87,8 @@ private String getRequestBody(final HttpServletRequest request) throws IOExcepti return builder.toString(); } - static Map parseUrlEncoded(String payload) { - List kvPairs = Arrays.asList(payload.split("\\&")); + List kvPairs = Arrays.asList(payload.split("&")); return kvPairs.stream() .map((kvPair) -> { @@ -119,11 +102,7 @@ static Map parseUrlEncoded(String payload) { } return List.of(name, value); - }) - .collect(toMap( - (tuple) -> tuple.get(0), - (tuple) -> tuple.get(1) - )); + }).collect(toMap((tuple) -> tuple.get(0), (tuple) -> tuple.get(1))); } private String getFullRequestUrl(HttpServletRequest request) { @@ -141,5 +120,6 @@ public void destroy() { producer.close(Duration.ofSeconds(10)); } } + } diff --git a/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/model/SendMessageRequest.java b/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/dto/SendMessageRequest.java similarity index 87% rename from backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/model/SendMessageRequest.java rename to backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/dto/SendMessageRequest.java index 3ad19e8ac2..19ce73a3d5 100644 --- a/backend/sources/facebook/sender/src/main/java/co/airy/core/sources/facebook/model/SendMessageRequest.java +++ b/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/dto/SendMessageRequest.java @@ -1,4 +1,4 @@ -package co.airy.core.sources.facebook.model; +package co.airy.core.sources.twilio.dto; import co.airy.avro.communication.Channel; import co.airy.avro.communication.Message; @@ -10,7 +10,7 @@ import java.io.Serializable; @Data -@Builder +@Builder(toBuilder = true) @NoArgsConstructor @AllArgsConstructor public class SendMessageRequest implements Serializable { diff --git a/backend/sources/twilio/sender/src/main/java/co/airy/core/sources/twilio/services/Api.java b/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/services/Api.java similarity index 86% rename from backend/sources/twilio/sender/src/main/java/co/airy/core/sources/twilio/services/Api.java rename to backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/services/Api.java index 050016f795..de9f6cbfd0 100644 --- a/backend/sources/twilio/sender/src/main/java/co/airy/core/sources/twilio/services/Api.java +++ b/backend/sources/twilio/connector/src/main/java/co/airy/core/sources/twilio/services/Api.java @@ -12,7 +12,7 @@ public class Api { private final String authToken; private final String accountSid; - public Api(@Value("${twilio.auth_token}") String authToken, @Value("${twilio.account_sid}") String accountSid) { + public Api(@Value("${twilio.auth-token}") String authToken, @Value("${twilio.account-sid}") String accountSid) { this.authToken = authToken; this.accountSid = accountSid; } diff --git a/backend/sources/twilio/connector/src/main/resources/application.properties b/backend/sources/twilio/connector/src/main/resources/application.properties new file mode 100644 index 0000000000..a121db2adb --- /dev/null +++ b/backend/sources/twilio/connector/src/main/resources/application.properties @@ -0,0 +1,6 @@ +auth.jwt-secret=${JWT_SECRET} +kafka.brokers=${KAFKA_BROKERS} +kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} +kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS} +twilio.account-sid=${TWILIO_ACCOUNT_SID} +twilio.auth-token=${TWILIO_AUTH_TOKEN} diff --git a/backend/sources/twilio/sender/src/test/java/co/airy/core/sources/twilio/SendMessageTest.java b/backend/sources/twilio/connector/src/test/java/co/airy/core/sources/twilio/SendMessageTest.java similarity index 94% rename from backend/sources/twilio/sender/src/test/java/co/airy/core/sources/twilio/SendMessageTest.java rename to backend/sources/twilio/connector/src/test/java/co/airy/core/sources/twilio/SendMessageTest.java index d50d227eb5..5e1c2919e2 100644 --- a/backend/sources/twilio/sender/src/test/java/co/airy/core/sources/twilio/SendMessageTest.java +++ b/backend/sources/twilio/connector/src/test/java/co/airy/core/sources/twilio/SendMessageTest.java @@ -25,6 +25,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import java.time.Instant; @@ -37,12 +38,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.doNothing; -@SpringBootTest(properties = { - "kafka.cleanup=true", - "kafka.commit-interval-ms=100", - "twilio.account_sid=12345", - "twilio.auth_token=token" -}, classes = AirySpringBootApplication.class) +@SpringBootTest(classes = AirySpringBootApplication.class) +@TestPropertySource(value = "classpath:test.properties") @ExtendWith(SpringExtension.class) class SendMessageTest { @@ -58,7 +55,10 @@ class SendMessageTest { @Autowired @InjectMocks - private Sender worker; + private Connector worker; + + @Autowired + private Stores stores; @BeforeAll static void beforeAll() throws Exception { @@ -78,7 +78,7 @@ static void afterAll() throws Exception { @BeforeEach void beforeEach() throws InterruptedException { MockitoAnnotations.initMocks(this); - retryOnException(() -> assertEquals(worker.getStreamState(), RUNNING), "Failed to reach RUNNING state."); + retryOnException(() -> assertEquals(stores.getStreamState(), RUNNING), "Failed to reach RUNNING state."); } @Test diff --git a/backend/sources/twilio/webhook/src/test/java/co/airy/core/sources/twilio/TwilioControllerIntegrationTest.java b/backend/sources/twilio/connector/src/test/java/co/airy/core/sources/twilio/WebhookControllerTest.java similarity index 88% rename from backend/sources/twilio/webhook/src/test/java/co/airy/core/sources/twilio/TwilioControllerIntegrationTest.java rename to backend/sources/twilio/connector/src/test/java/co/airy/core/sources/twilio/WebhookControllerTest.java index 2a4123e38b..8c4ff88df8 100644 --- a/backend/sources/twilio/webhook/src/test/java/co/airy/core/sources/twilio/TwilioControllerIntegrationTest.java +++ b/backend/sources/twilio/connector/src/test/java/co/airy/core/sources/twilio/WebhookControllerTest.java @@ -14,6 +14,7 @@ import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.MediaType; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.web.servlet.MockMvc; @@ -31,13 +32,11 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = AirySpringBootApplication.class, - properties = { - "twilio.token=whatever", - }) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = AirySpringBootApplication.class) +@TestPropertySource(value = "classpath:test.properties") @AutoConfigureMockMvc @ExtendWith(SpringExtension.class) -class TwilioControllerIntegrationTest { +class WebhookControllerTest { @RegisterExtension public static final SharedKafkaTestResource sharedKafkaTestResource = new SharedKafkaTestResource(); private static KafkaTestHelper testHelper; @@ -46,8 +45,8 @@ class TwilioControllerIntegrationTest { @Autowired private MockMvc mvc; - @Value("${twilio.token}") - private String twilioAuthToken; + @Value("${twilio.auth-token}") + private String authToken; @BeforeAll static void beforeAll() throws Exception { @@ -55,6 +54,11 @@ static void beforeAll() throws Exception { testHelper.beforeAll(); } + @AfterAll + static void afterAll() throws Exception { + testHelper.afterAll(); + } + @Test void failsForUnauthenticated() throws Exception { final Map parameters = Map.of( @@ -62,17 +66,15 @@ void failsForUnauthenticated() throws Exception { "SmsSid", "MM3753d789407e9d3f4c0a8b919ec5473f" ); - final String validationSignature = getValidationSignature("http://localhost/twilio", parameters, "DROP TABLE"); + final String validationSignature = getValidationSignature(parameters, "DROP TABLE"); mvc.perform(post("/twilio") .contentType(MediaType.APPLICATION_FORM_URLENCODED_VALUE) .header("X-Twilio-Signature", validationSignature) .content("ApiVersion=2010-04-01&SmsSid=MM3753d789407e9d3f4c0a8b919ec5473f") - ) - .andExpect(status().isForbidden()); + ).andExpect(status().isForbidden()); } - @Test void canAcceptRequests() throws Exception { final Map parameters = Map.of( @@ -80,7 +82,7 @@ void canAcceptRequests() throws Exception { "SmsSid", "MM3753d789407e9d3f4c0a8b919ec5473f" ); - final String validationSignature = getValidationSignature("http://localhost/twilio", parameters, twilioAuthToken); + final String validationSignature = getValidationSignature(parameters, authToken); mvc.perform(post("/twilio") .contentType(MediaType.APPLICATION_FORM_URLENCODED_VALUE) @@ -97,11 +99,11 @@ void canAcceptRequests() throws Exception { private static final String HMAC = "HmacSHA1"; // Adapted from com.twilio.security.RequestValidator only for testing - private String getValidationSignature(String url, Map params, String authToken) { + private String getValidationSignature(Map params, String authToken) { try { final SecretKeySpec signingKey = new SecretKeySpec(authToken.getBytes(), HMAC); - StringBuilder builder = new StringBuilder(url); + StringBuilder builder = new StringBuilder("http://localhost/twilio"); if (params != null) { List sortedKeys = new ArrayList<>(params.keySet()); Collections.sort(sortedKeys); @@ -124,9 +126,4 @@ private String getValidationSignature(String url, Map params, St return null; } } - - @AfterAll - static void afterAll() throws Exception { - testHelper.afterAll(); - } } diff --git a/backend/sources/twilio/connector/src/test/resources/test.properties b/backend/sources/twilio/connector/src/test/resources/test.properties new file mode 100644 index 0000000000..a18982058d --- /dev/null +++ b/backend/sources/twilio/connector/src/test/resources/test.properties @@ -0,0 +1,5 @@ +kafka.cleanup=true +kafka.commit-interval-ms=100 +twilio.account-sid=42 +twilio.auth-token=token +auth.jwt-secret=42424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242424242 \ No newline at end of file diff --git a/backend/sources/twilio/events-router/BUILD b/backend/sources/twilio/events-router/BUILD index e3c6f1c1f7..7e19bb8e3d 100644 --- a/backend/sources/twilio/events-router/BUILD +++ b/backend/sources/twilio/events-router/BUILD @@ -4,10 +4,9 @@ load("//tools/build:container_push.bzl", "container_push") app_deps = [ "//backend:base_app", - "//backend:channel", - "//backend:message", + "//backend/model/channel:channel", + "//backend/model/message:message", "//lib/java/uuid", - "//lib/java/payload", "//lib/java/log", "//lib/java/kafka/schema:source-twilio-events", "//lib/java/spring/kafka/core:spring-kafka-core", diff --git a/backend/sources/twilio/events-router/src/main/java/co/airy/core/sources/twilio/TwilioInfoExtractor.java b/backend/sources/twilio/events-router/src/main/java/co/airy/core/sources/twilio/TwilioInfoExtractor.java index e23a506576..fbedc96a9b 100644 --- a/backend/sources/twilio/events-router/src/main/java/co/airy/core/sources/twilio/TwilioInfoExtractor.java +++ b/backend/sources/twilio/events-router/src/main/java/co/airy/core/sources/twilio/TwilioInfoExtractor.java @@ -23,7 +23,7 @@ static TwilioEventInfo extract(String payload) { } static Map parseUrlEncoded(String payload) { - List kvPairs = Arrays.asList(payload.split("\\&")); + List kvPairs = Arrays.asList(payload.split("&")); return kvPairs.stream() .map((kvPair) -> { @@ -39,9 +39,6 @@ static Map parseUrlEncoded(String payload) { return List.of(name, value); }) .filter(Objects::nonNull) - .collect(toMap( - (tuple) -> tuple.get(0), - (tuple) -> tuple.get(1) - )); + .collect(toMap((tuple) -> tuple.get(0), (tuple) -> tuple.get(1))); } } diff --git a/backend/sources/twilio/events-router/src/main/resources/application.properties b/backend/sources/twilio/events-router/src/main/resources/application.properties index cfeec4b55c..3c1d68e8b2 100644 --- a/backend/sources/twilio/events-router/src/main/resources/application.properties +++ b/backend/sources/twilio/events-router/src/main/resources/application.properties @@ -1,2 +1,3 @@ kafka.brokers=${KAFKA_BROKERS} kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} +kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS} diff --git a/backend/sources/twilio/events-router/src/test/java/co/airy/core/sources/twilio/EventsRouterTest.java b/backend/sources/twilio/events-router/src/test/java/co/airy/core/sources/twilio/EventsRouterTest.java index 7bf7db3dc9..0cb012b694 100644 --- a/backend/sources/twilio/events-router/src/test/java/co/airy/core/sources/twilio/EventsRouterTest.java +++ b/backend/sources/twilio/events-router/src/test/java/co/airy/core/sources/twilio/EventsRouterTest.java @@ -9,7 +9,6 @@ import co.airy.kafka.test.KafkaTestHelper; import co.airy.kafka.test.junit.SharedKafkaTestResource; import co.airy.spring.core.AirySpringBootApplication; -import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.ProducerRecord; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -19,6 +18,7 @@ import org.junit.jupiter.api.extension.RegisterExtension; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import java.util.List; @@ -29,11 +29,8 @@ import static org.apache.kafka.streams.KafkaStreams.State.RUNNING; import static org.junit.jupiter.api.Assertions.assertEquals; -@Slf4j -@SpringBootTest(properties = { - "kafka.cleanup=true", - "kafka.commit-interval-ms=100" -}, classes = AirySpringBootApplication.class) +@SpringBootTest(classes = AirySpringBootApplication.class) +@TestPropertySource(value = "classpath:test.properties") @ExtendWith(SpringExtension.class) class EventsRouterTest { @RegisterExtension @@ -96,10 +93,9 @@ void canRouteTwilioMessage() throws Exception { String broken = "{\"wait\":\"this isn't url encoded ._.\"}"; - testHelper.produceRecords( - List.of( - new ProducerRecord<>(sourceTwilioEvents.name(), UUID.randomUUID().toString(), event), - new ProducerRecord<>(sourceTwilioEvents.name(), UUID.randomUUID().toString(), broken) + testHelper.produceRecords(List.of( + new ProducerRecord<>(sourceTwilioEvents.name(), UUID.randomUUID().toString(), event), + new ProducerRecord<>(sourceTwilioEvents.name(), UUID.randomUUID().toString(), broken) ) ); diff --git a/backend/sources/twilio/events-router/src/test/resources/test.properties b/backend/sources/twilio/events-router/src/test/resources/test.properties new file mode 100644 index 0000000000..4ff2a34427 --- /dev/null +++ b/backend/sources/twilio/events-router/src/test/resources/test.properties @@ -0,0 +1,2 @@ +kafka.cleanup=true +kafka.commit-interval-ms=100 \ No newline at end of file diff --git a/backend/sources/twilio/sender/src/main/resources/application.properties b/backend/sources/twilio/sender/src/main/resources/application.properties deleted file mode 100644 index 8bd97aebbe..0000000000 --- a/backend/sources/twilio/sender/src/main/resources/application.properties +++ /dev/null @@ -1,4 +0,0 @@ -kafka.brokers=${KAFKA_BROKERS} -kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} -twilio.auth_token=${TWILIO_AUTH_TOKEN} -twilio.account_sid=${TWILIO_ACCOUNT_SID} diff --git a/backend/sources/twilio/webhook/BUILD b/backend/sources/twilio/webhook/BUILD deleted file mode 100644 index 4ea4d399c2..0000000000 --- a/backend/sources/twilio/webhook/BUILD +++ /dev/null @@ -1,39 +0,0 @@ -load("//tools/build:springboot.bzl", "springboot") -load("//tools/build:junit5.bzl", "junit5") -load("//tools/build:container_push.bzl", "container_push") - -app_deps = [ - "//backend:base_app", - "//:springboot_actuator", - "@maven//:com_twilio_sdk_twilio", - "//lib/java/kafka/schema:source-twilio-events", - "//lib/java/spring/kafka/core:spring-kafka-core", - "//lib/java/spring/kafka/healthcheck", -] - -springboot( - name = "webhook", - srcs = glob(["src/main/java/**/*.java"]), - main_class = "co.airy.spring.core.AirySpringBootApplication", - deps = app_deps, -) - -[ - junit5( - size = "small", - file = file, - resources = glob(["src/test/resources/**/*"]), - deps = [ - ":app", - "//backend:base_test", - "@maven//:javax_xml_bind_jaxb_api", - "//lib/java/kafka/test:kafka-test", - ] + app_deps, - ) - for file in glob(["src/test/java/**/*Test.java"]) -] - -container_push( - registry = "ghcr.io/airyhq/sources", - repository = "twilio-webhook", -) diff --git a/backend/sources/twilio/webhook/src/main/resources/application.properties b/backend/sources/twilio/webhook/src/main/resources/application.properties deleted file mode 100644 index b5efdecdd9..0000000000 --- a/backend/sources/twilio/webhook/src/main/resources/application.properties +++ /dev/null @@ -1,3 +0,0 @@ -kafka.brokers=${KAFKA_BROKERS} -kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} -twilio.token=${TWILIO_AUTH_TOKEN} diff --git a/backend/webhook/publisher/BUILD b/backend/webhook/publisher/BUILD index bfcd630b4f..01e02927ee 100644 --- a/backend/webhook/publisher/BUILD +++ b/backend/webhook/publisher/BUILD @@ -4,11 +4,11 @@ load("//tools/build:container_push.bzl", "container_push") app_deps = [ "//backend:base_app", - "//backend:message", + "//backend/model/message:message", "//backend:webhook", "//lib/java/uuid", - "//lib/java/payload", "//lib/java/mapping", + "//lib/java/date", "//lib/java/spring/kafka/core:spring-kafka-core", "//lib/java/spring/kafka/streams:spring-kafka-streams", "@maven//:io_lettuce_lettuce_core", @@ -16,7 +16,7 @@ app_deps = [ ] springboot( - name = "events-router", + name = "webhook-publisher", srcs = glob(["src/main/java/**/*.java"]), main_class = "co.airy.spring.core.AirySpringBootApplication", deps = app_deps, diff --git a/backend/webhook/publisher/src/main/java/co/airy/core/webhook/publisher/Mapper.java b/backend/webhook/publisher/src/main/java/co/airy/core/webhook/publisher/Mapper.java index 3519f57731..8ccffff625 100644 --- a/backend/webhook/publisher/src/main/java/co/airy/core/webhook/publisher/Mapper.java +++ b/backend/webhook/publisher/src/main/java/co/airy/core/webhook/publisher/Mapper.java @@ -8,9 +8,10 @@ import co.airy.mapping.model.Text; import org.springframework.stereotype.Component; +import java.util.List; import java.util.Map; -import static co.airy.payload.format.DateFormat.isoFromMillis; +import static co.airy.date.format.DateFormat.isoFromMillis; @Component public class Mapper { @@ -21,16 +22,21 @@ public class Mapper { } public WebhookBody fromMessage(Message message) throws Exception { - final Content content = contentMapper.renderWithDefaultAndLog(message); + final List content = contentMapper.renderWithDefaultAndLog(message); - if (!(content instanceof Text)) { + final Text textContent = (Text) content.stream() + .filter(c -> (c instanceof Text)) + .findFirst() + .orElse(null); + + if (textContent == null) { throw new NotATextMessage(); } return WebhookBody.builder() .conversationId(message.getConversationId()) .id(message.getId()) - .text(((Text) content).getText()) + .text(textContent.getText()) .source(message.getSource()) .postback(buildPostback(message)) .sentAt(isoFromMillis(message.getSentAt())) diff --git a/backend/webhook/publisher/src/main/resources/application.properties b/backend/webhook/publisher/src/main/resources/application.properties index b214ccd5b4..dc8be7692e 100644 --- a/backend/webhook/publisher/src/main/resources/application.properties +++ b/backend/webhook/publisher/src/main/resources/application.properties @@ -1,7 +1,7 @@ kafka.brokers=${KAFKA_BROKERS} kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} kafka.cleanup=${KAFKA_CLEANUP:false} -kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS:30000} +kafka.commit-interval-ms=${KAFKA_COMMIT_INTERVAL_MS} redis.url=${REDIS_HOSTNAME} redis.port=${REDIS_PORT} diff --git a/backend/webhook/publisher/src/test/java/co/airy/core/webhook/publisher/PublisherTest.java b/backend/webhook/publisher/src/test/java/co/airy/core/webhook/publisher/PublisherTest.java index 5d649eacc2..23760c34ba 100644 --- a/backend/webhook/publisher/src/test/java/co/airy/core/webhook/publisher/PublisherTest.java +++ b/backend/webhook/publisher/src/test/java/co/airy/core/webhook/publisher/PublisherTest.java @@ -15,7 +15,6 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; @@ -26,6 +25,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import java.time.Instant; @@ -39,14 +39,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.doNothing; -@Tag("kafka-integration") -@SpringBootTest(properties = { - "kafka.cleanup=true", - "kafka.cache.max.bytes=0", - "kafka.commit-interval-ms=100", - "redis.url=no", - "redis.port=10", -}, classes = AirySpringBootApplication.class) +@SpringBootTest(classes = AirySpringBootApplication.class) +@TestPropertySource(value = "classpath:test.properties") @ExtendWith(SpringExtension.class) public class PublisherTest { @RegisterExtension @@ -121,8 +115,7 @@ void canPublishMessageToQueue() throws Exception { .setConversationId("conversationId") .setChannelId("channelId") .setContent("{\"text\":\"hello world\"}") - .build()) - ); + .build())); // Don't publish the message update messages.add(new ProducerRecord<>(applicationCommunicationMessages.name(), messageId, @@ -143,9 +136,6 @@ void canPublishMessageToQueue() throws Exception { kafkaTestHelper.produceRecords(messages); - retryOnException(() -> { - final List allMessages = batchCaptor.getAllValues(); - assertEquals(4, allMessages.size()); - }, "Number of delivered message is incorrect"); + retryOnException(() -> assertEquals(4, batchCaptor.getAllValues().size()), "Number of delivered message is incorrect"); } } diff --git a/backend/webhook/publisher/src/test/resources/test.properties b/backend/webhook/publisher/src/test/resources/test.properties new file mode 100644 index 0000000000..160ec0212b --- /dev/null +++ b/backend/webhook/publisher/src/test/resources/test.properties @@ -0,0 +1,5 @@ +kafka.cleanup=true +kafka.cache.max.bytes=0 +kafka.commit-interval-ms=100 +redis.url=no +redis.port=10 \ No newline at end of file diff --git a/docs/docs/api/http.md b/docs/docs/api/http.md index c66f39d541..7e14f9814d 100644 --- a/docs/docs/api/http.md +++ b/docs/docs/api/http.md @@ -19,8 +19,8 @@ The HTTP endpoints adhere to the following conventions: In order to communicate with the API endpoints, you need a valid [JWT](https://jwt.io/) token. Get a valid token by sending a request to the -login endpoint [login](#login). It returns short-lived JWT token you can use for -API requests. +login endpoint [login](#login). It returns a short-lived JWT token you can use +for HTTP requests. ### Login @@ -87,7 +87,7 @@ The password **must** be at least 6 characters long. } ``` -This endpoint returns the same response as the login. +This endpoint returns the same response as `POST /login`. ### Conversations @@ -109,7 +109,7 @@ class](https://github.com/airyhq/airy/blob/main/backend/api/communication/src/ma **Sample request** -Find all users with the last name "Lovelace": +Find users whose name ends with "Lovelace": ```json5 { @@ -136,8 +136,8 @@ Find all users with the last name "Lovelace": { "id": "a688d36c-a85e-44af-bc02-4248c2c97622", "channel": { - "name": "facebook", - // name of the source + "name": "Facebook page name", + "source": "facebook", "id": "318efa04-7cc1-4200-988e-50699d0dd6e3" }, "created_at": "2019-01-07T09:01:44.000Z", @@ -151,11 +151,13 @@ Find all users with the last name "Lovelace": "tags": ["f339c325-8614-43cb-a70a-e83d81bf56fc"], "last_message": { id: "{UUID}", - content: { - text: "{String}", - type: "text" - // Determines the schema of the content - }, + content: [ + { + text: "{String}", + type: "text" + // Determines the schema of the content + }, + ] // typed source message model state: "{String}", // delivery state of message, one of PENDING, FAILED, DELIVERED @@ -211,11 +213,13 @@ Find all users with the last name "Lovelace": "tags": ["f339c325-8614-43cb-a70a-e83d81bf56fc"], "last_message": { "id": "{UUID}", - "content": { - "text": "{String}", - "type": "text" - // Determines the schema of the content - }, + "content": [ + { + "text": "{String}", + "type": "text" + // Determines the schema of the content + } + ], // typed source message model "delivery_state": "{String}", // delivery state of message, one of PENDING, FAILED, DELIVERED @@ -250,7 +254,8 @@ Resets the unread count of a conversation and returns status code `202 (Accepted #### Tag a conversation -Tags an existing conversation with an existing tag. Returns status code `200` if successful. +Tags an existing conversation with [an existing tag](#creating-a-tag). Returns +status code `200` if successful. `POST /conversations.tag` @@ -316,11 +321,13 @@ This is a [paginated](#pagination) endpoint. Messages are sorted from oldest to "data": [ { "id": "{UUID}", - "content": { - "text": "{String}", - "type": "text" - // Determines the schema of the content - }, + "content": [ + { + "text": "{String}", + "type": "text" + // Determines the schema of the content + } + ], // typed source message model "state": "{String}", // delivery state of message, one of PENDING, FAILED, DELIVERED @@ -361,11 +368,13 @@ Sends a message to a conversation and returns a payload. ```json5 { "id": "{UUID}", - "content": { - "text": "{String}", - "type": "text" - // Determines the schema of the content - }, + "content": [ + { + "text": "{String}", + "type": "text" + // Determines the schema of the content + } + ], // typed source message model "state": "{String}", // delivery state of message, one of PENDING, FAILED, DELIVERED @@ -378,86 +387,8 @@ Sends a message to a conversation and returns a payload. ### Channels -#### Connecting channels - -`POST /channels.connect` - -A synchronous endpoint that makes a request to the source -to connect the channel. - -This action is idempotent, so if the channel is already connected, the request returns status code `202`. - -Connecting a channel is source-specific by nature, refer to the relevant documentation for the correct payload: - -- [Facebook](/sources/facebook.md#connecting-a-channel) -- [Google](/sources/google.md#connecting-a-channel) -- [SMS - Twilio](/sources/sms-twilio.md#connecting-a-channel) -- [WhatsApp - Twilio](/sources/whatsapp-twilio.md#connecting-a-channel) - -#### Disconnecting Channels - -`POST /channels.disconnect` - -A synchronous endpoint that makes a request to the source -to disconnect the channel. It marks the channel as disconnected and deletes the -auth token. - -This action is idempotent, so if the channel is disconnected, the request returns status code `202`. -If the channel is unknown, the request returns status code `400`. - -**Sample request** - -```json5 -{ - "channel_id": "uuid" -} -``` - -#### Explore channels - -`POST /channels.explore` - -A synchronous endpoint that makes a request to the source -to list all the available channels. Some of those channels may already -be connected, which is accounted for in the boolean field `connected`. Due to -the nature of the request, the response time may vary. - - - -The request requires an authentication `token`, which has a different meaning for each source: - -- `facebook` The user access token - -**Sample request** - -```json5 -{ - "source": "facebook", - "token": "authentication token" -} -``` - -**Sample response** - -```json5 -{ - "data": [ - { - "name": "my page 1", - "source": "facebook", - "source_channel_id": "fb-page-id-1", - "connected": false, - "image_url": "http://example.org/avatar.jpeg" // optional - }, - { - "name": "my page 2", - "source": "facebook", - "source_channel_id": "fb-page-id-2", - "connected": true - } - ] -} -``` +Please refer to our [channel](glossary.md#channel) definition for more +information. #### List channels @@ -487,6 +418,9 @@ The request requires an authentication `token`, which has a different meaning fo ### Tags +Please refer to our [tag](glossary.md#tag) definition for more +information. + #### Creating a tag `POST /tags.create` @@ -570,6 +504,38 @@ If action is successful, returns HTTP status `200`. } ``` +### Metadata + +Refer to our [metadata](glossary.md#metadata) definition for more +information. + +### Setting metadata + +`POST /metadata.set` + +```json +{ + "conversation_id": "conversation-id", + "key": "ad.id", + "value": "Grace" +} +``` + +The endpoint returns status code `200` if the operation was successful, and `400` if not. + +### Removing metadata + +`POST /metadata.remove` + +```json +{ + "conversation_id": "conversation-id", + "key": "ad.id" +} +``` + +This endpoint returns status code `200` if the operation was successful, and `500` if not. + ## Pagination By default, paginated endpoints return a maximum of 20 elements on the first page. @@ -620,41 +586,9 @@ The response comes in two parts: - `filtered_total` The total number of elements across pages in the context of the current - filter selection. Only applicable to paginated endpoints that accept filter - input. + filter selection. Only applicable to paginated endpoints that can filter + data. - `total` The total number of elements across all pages. - -### Metadata - -Refer to our [metadata](glossary.md#metadata) definition for more -information. - -### Setting metadata - -`POST /metadata.set` - -```json -{ - "conversation_id": "conversation-id", - "key": "ad.id", - "value": "Grace" -} -``` - -The endpoint returns status code `200` if the operation was successful, and `400` if not. - -### Removing metadata - -`POST /metadata.remove` - -```json -{ - "conversation_id": "conversation-id", - "key": "ad.id" -} -``` - -This endpoint returns status code `200` if the operation was successful, and `500` if not. diff --git a/docs/docs/glossary.md b/docs/docs/glossary.md index f40643dba0..26f8c25169 100644 --- a/docs/docs/glossary.md +++ b/docs/docs/glossary.md @@ -31,8 +31,8 @@ Platform. ## Contact A contact represents the [source](#source) participant. A -[conversation](#conversation) exists _only_ if it has _at least one_ message -from a contact. +[conversation](#conversation) exists _only_ if it has _at least one_ +[message](#message) from a contact. ## Conversation @@ -115,6 +115,13 @@ e.g. | "sender.id" | "123A" | | "sender.contact.first_name | "Grace" | +### Tag + +A tag is a specialized metadata, which is used to tag +[conversations](#conversation). As the use case of tagging conversations is so +common, the Airy Core Platform provides specialized endpoints and filters for +tagging conversations. + ## Source A source represents a system that generates messaging data that a user wants to @@ -122,9 +129,9 @@ process with the Airy Core Platform. ### Provider -Source providers are API platforms that allow the Airy Core Platform to connect to -one or more of their sources typically via a webhook. E.g. Twilio is a source provider -for the Twilio SMS and Whatsapp sources. +Source providers are API platforms that allow the Airy Core Platform to connect +to one or more of their sources typically via a webhook. E.g. Twilio is a source +provider for the Twilio SMS and WhatsApp sources. ## User diff --git a/docs/docs/guides/airy-core-and-rasa.md b/docs/docs/guides/airy-core-and-rasa.md index 2bb8d82fce..d418115411 100644 --- a/docs/docs/guides/airy-core-and-rasa.md +++ b/docs/docs/guides/airy-core-and-rasa.md @@ -95,7 +95,7 @@ Now you should have a working integration 🎉. To test the connection write a message to one of your channels: The Airy Core Platform will forward it to your Rasa installation, which will respond using the -Airy Core Platform API. This is what you will see in the logs of the demo +Airy Core Platform API. This is what you should see in the logs of the demo repository: send message successful connection log diff --git a/docs/docs/guides/airy-core-in-production.md b/docs/docs/guides/airy-core-in-production.md index 9cf50e38fb..283e556df0 100644 --- a/docs/docs/guides/airy-core-in-production.md +++ b/docs/docs/guides/airy-core-in-production.md @@ -29,7 +29,8 @@ Schema registry servers. We recommend using at least five Kafka brokers and set the replication factor of _all_ topics to `3`. So that even if two brokers would become unavailable at the same time, the system would still function. -We also recommend running at least three Zookeepers and two instances of the Confluent Schema registry. +We also recommend running at least three Zookeepers and two instances of the +Confluent Schema registry. If you decide to run the Kafka cluster on Kubernetes, we recommend running Kafka and Zookeeper as StatefulSet workloads, for persistent storage. The Confluent @@ -51,7 +52,8 @@ The Kafka cluster is usually started in the following order: parameter `kafkastore.bootstrap.server` of the configuration file `/etc/schema-registry/schema-registry.properties`. -The location of the configuration files can vary depending on the particular installation. +The location of the configuration files can vary depending on the particular +installation. Once the Kafka cluster is up and running, the required topics must be created. You can find them in the Bash script @@ -61,11 +63,18 @@ following environment variables to run: - `ZOOKEEPER` (default: airy-cp-zookeeper:2181) - `PARTITIONS` (default: 10) - `REPLICAS` (default: 1) +- `AIRY_CORE_NAMESPACE` (default: ''). Helpful to namespace your topics in case + you are installing the Airy Core Platform in an existing Kafka cluster We do not recommend running Kafka on docker for production environments. However, we provide a way to deploy the whole Kafka cluster on top of Kubernetes with Helm as we use this approach for test installations. +The default commit interval is set to 1000 ms (1 second). This is _not_ recommended +for production usage. +You change the `commitInterval` to a more suitable production value in the configuration file +`infrastructure/helm-chart/charts/apps/charts/airy-config/values.yaml`. + To deploy Kafka on Kubernetes with Helm, you can run: ```sh diff --git a/docs/docs/guides/airy-core-in-test-env.md b/docs/docs/guides/airy-core-in-test-env.md index 6d9f2b1ebb..679d505a28 100644 --- a/docs/docs/guides/airy-core-in-test-env.md +++ b/docs/docs/guides/airy-core-in-test-env.md @@ -115,7 +115,7 @@ then provides a reversed proxy connectivity back to the webhook services, running inside the Kubernetes cluster. By default, the ngrok client is configured to use the ngrok server created by -Airy and running on https://tunnel.airy.co. This configuration is specified in +Airy and run on https://tunnel.airy.co. This configuration is specified in the `ngrok-client-config` ConfigMap. ``` @@ -183,8 +183,8 @@ vagrant destroy ## Known Issues -If you have just installed Virtualbox and see this error during the bootstrap -you should [give Virtualbox +If you have just installed VirtualBox and see this error during the bootstrap +you should [give VirtualBox permissions](https://www.howtogeek.com/658047/how-to-fix-virtualboxs-%E2%80%9Ckernel-driver-not-installed-rc-1908-error/). ``` diff --git a/docs/docs/index.md b/docs/docs/index.md index 15c9896439..e654937505 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -17,17 +17,17 @@ following components: independent contacts, conversations, and messages (see our [glossary](glossary.md) for formal definitions). -- An [HTTP api](api/http.md) that allows to manage the data sets the platform - handles. +- An [HTTP API](api/http.md) that allows you to manage the data sets the + platform handles. - A [webhook](api/webhook) integration server that allows to programmatically participate in conversations by sending messages. The webhook integration - exposes events users can "listen" to and react programmatically. + exposes events you can "listen" to and react programmatically. -- A [WebSocket](api/websocket) server that allows to receive near real-time updates about - the data flowing through the system. +- A [WebSocket](api/websocket) server that allows you to receive near real-time + updates about the data flowing through the system. -## Bootstrapping the Airy Core Platform +## Bootstrap the Airy Core Platform Run the Airy Core Platform locally by entering the following commands: @@ -59,7 +59,7 @@ hosts file yourself. Check out our [guide for running in test environment](guides/airy-core-in-test-env.md) for detailed information. -## Connecting a chat plugin source +## Connect a Chat Plugin source The chat plugin source is well suited for a first integration because it does not require any configuration. @@ -74,11 +74,9 @@ token=$(echo $(curl -H 'Content-Type: application/json' -d \ \"password\":\"the_answer_is_42\" \ }" api.airy/users.login) | jq -r '.token') curl -H "Content-Type: application/json" -H "Authorization: $token" -d \ -"{ \ - \"source\": \"chat_plugin\", \ - \"source_channel_id\": \"my-chat-channel-1\", \ +"{ \"name\": \"chat plugin source\" -}" api.airy/channels.connect +}" api.airy/chatplugin.connect ``` channels_connect @@ -86,7 +84,7 @@ curl -H "Content-Type: application/json" -H "Authorization: $token" -d \ The ID from the response is the `channel_id`. It is required for the next steps, so note it down. -## Sending messages with the chat plugin +## Send messages via the Chat Plugin Pass the `channel_id` as a query parameter when opening the demo page in your browser. This authenticates the chat plugin and enables you to send messages diff --git a/docs/docs/overview/architecture.md b/docs/docs/overview/architecture.md index f20d8f352a..a7a6e4b20f 100644 --- a/docs/docs/overview/architecture.md +++ b/docs/docs/overview/architecture.md @@ -1,5 +1,5 @@ --- -title: The Airy Core Plaform architecture +title: The Airy Core Platform architecture sidebar_label: Architecture --- @@ -25,7 +25,7 @@ which run as part of the Airy Core Platform: - sources-`SOURCE_NAME`-webhook - Ingest events from the `SOURCE_NAME` source - sources-`SOURCE_NAME`-events-router - Process messages from a `SOURCE_NAME` source -- sources-`SOURCE_NAME`-sender - Send events (mostly messages) to a `SOURCE_NAME` source +- sources-`SOURCE_NAME`-connector - Send events (mostly messages) to a `SOURCE_NAME` source and extracts metadata ## API @@ -35,8 +35,9 @@ which run as part of the Airy Core Platform: ## Webhook -- webhook-publisher - Process conversational data and write in Redis the events to be exposed to external parties. -- webhook-consumer - Read from Redis and send events to external webhooks +- webhook-publisher - Processes conversational data and write in Redis the events + to be exposed to external parties. +- webhook-consumer - Reads from Redis and send events to external webhooks ## Frontend diff --git a/docs/docs/overview/kafka.md b/docs/docs/overview/kafka.md index f2fa2661d6..e81b9fd1a6 100644 --- a/docs/docs/overview/kafka.md +++ b/docs/docs/overview/kafka.md @@ -9,7 +9,7 @@ the Airy Core Platform. ## Topic naming conventions Inspired by [this -article](https://medium.com/@criccomini/how-to-paint-a-bike-shed-kafka-topic-naming-conventions-1b7259790073), +article](https://riccomini.name/how-paint-bike-shed-kafka-topic-naming-conventions), our naming conventions follow these rules: - A topic has a three-part name: `..` @@ -21,19 +21,14 @@ Each part defines a more granular scope: - `kind` is the type of data the topic contains at the highest level possible. Valid examples are: `etl`, `logging`, `tracking`. - `domain` is what you would call a database name in a traditional - rdms. -- `dataset` is what you would call a database table in a traditional rdms. + RDMS. +- `dataset` is what you would call a database table in a traditional RDMS. Given these rules, here are a few examples: ``` -tracking.user.clicks -tracking.page.views - -etl.billing.invalid-cc-cards -etl.billing.frauds - -application.entity.organizations -application.communication.conversations application.communication.messages +application.communication.metadata + +ops.application.health-checks ``` diff --git a/docs/docs/sources/channel-disconnect.mdx b/docs/docs/sources/channel-disconnect.mdx new file mode 100644 index 0000000000..5f9684eb18 --- /dev/null +++ b/docs/docs/sources/channel-disconnect.mdx @@ -0,0 +1,14 @@ +A synchronous endpoint that makes a request to the source to disconnect the +channel. It marks the channel as disconnected and deletes the auth token. + +This action is idempotent, so if the channel is disconnected, the request +returns status code `202`. If the channel is unknown, the request returns status +code `400`. + +**Sample request** + +```json5 +{ + channel_id: 'uuid', +} +``` diff --git a/docs/docs/sources/chat-plugin.md b/docs/docs/sources/chat-plugin.md index 848b7062fc..6b2c9d4648 100644 --- a/docs/docs/sources/chat-plugin.md +++ b/docs/docs/sources/chat-plugin.md @@ -3,50 +3,57 @@ title: Chat Plugin sidebar_label: Chat Plugin --- -The Airy Core chat plugin is a fully-featured [source](/glossary.md#source) -that enables conversations with anonymous website visitors through a web chat -plugin. +The Airy Core Chat Plugin is a fully-featured [source](/glossary.md#source) that +enables conversations with anonymous website visitors through a web chat plugin. :::tip What you will learn -- How to connect a chat plugin -- How to install the chat plugin web widget -- How to use the HTTP and WebSocket APIs that power the chat plugin +- How to connect a Chat Plugin +- How to install the Chat Plugin web widget +- How to use the HTTP and WebSocket APIs that power the Chat Plugin ::: -## Connect a channel +## Connect -Connects a chat plugin source to the Airy Core Platform. +Connects a Chat Plugin source to the Airy Core Platform. ``` -POST /channels.connect +POST /chatplugin.connect ``` -- `source` _must_ be `chat_plugin` -- `source_channel_id` is a unique identifier of your choice +- `name` is a unique identifier of your choice ```json5 { - "source": "chat_plugin", - "source_channel_id": "website-identifier-42" + "name": "website-identifier-42" } ``` -**Sample Response** +**Sample response** ```json5 { - "id": "channel-uuid-1", - "name": "Chat plugin", + "id": "1F679227-76C2-4302-BB12-703B2ADB0F66", + "name": "website-identifier-42", "source": "chat_plugin", - "source_channel_id": "awesome-website-42" + "source_channel_id": "website-identifier-42" } ``` +## Disconnect + +``` +POST /chatplugin.disconnect +``` + +import ChannelDisconnect from './channel-disconnect.mdx' + + + ## Installation -To install the chat plugin UI on your website add the following script tag to +To install the Chat Plugin UI on your website add the following script tag to the `` section: ```html @@ -66,7 +73,7 @@ the `` section: You must replace `CHANNEL_ID` with the channel id obtained when [connecting](#connecting-a-channel) the source and `SCRIPT_HOST` with the host -of your chat plugin server. When using the local vagrant environment +of your Chat Plugin server. When using the local vagrant environment `SCRIPT_HOST` must be set to `chatplugin.airy`. :::note @@ -91,7 +98,10 @@ API](api/http.md#introduction). The request returns an authentication token that needs to be included in the WebSocket connection handshake. -**Sample Request** +You can either pass the `channel_id` for a new conversation or a `resume_token` that was obtained in a +previous conversation using the [resume endpoint](#get-a-resume-token). + +**Sample request** ```json5 { @@ -99,11 +109,60 @@ WebSocket connection handshake. } ``` -**Sample Response** +**Sample response (New conversation)** + +```json5 +{ + "token": "jwt", + "messages": [] +} +``` + +**Sample response (Resumed conversation)** + +```json5 +{ + "token": "jwt", + "messages": [ + { + "id": "{UUID}", + "content": [ + { + "text": "{String}", + "type": "text" + // Determines the schema of the content + } + ], + // typed source message model + "state": "{String}", + // delivery state of message, one of PENDING, FAILED, DELIVERED + "sender_type": "{string/enum}", + // See glossary + "sent_at": "{string}" + //'yyyy-MM-dd'T'HH:mm:ss.SSSZ' date in UTC form, to be localized by clients + } + ] +} +``` + +### Get a resume token + +`POST /chatplugin.resumeToken` + +You must set the `token` obtained on the [authorization endpoint](#authenticating-web-users) as an `Authorization` +header. + +**Sample request** + +```json5 +{} +``` + +**Sample response** ```json5 { - "token": "jwt auth token" + "resume_token": "jwt auth token" } ``` @@ -114,7 +173,7 @@ header. `POST /chatplugin.send` -**Sample Request** +**Sample request** ```json5 { @@ -124,7 +183,7 @@ header. } ``` -**Sample Response** +**Sample response** ```json5 { @@ -155,7 +214,7 @@ The WebSocket connection endpoint is at `/ws.chatplugin`. `/user/queue/message` -**Sample Payload** +**Sample payload** ```json5 { diff --git a/docs/docs/sources/facebook.md b/docs/docs/sources/facebook.md index 94eb09b704..ba6ae36229 100644 --- a/docs/docs/sources/facebook.md +++ b/docs/docs/sources/facebook.md @@ -51,7 +51,7 @@ Refer to the [test](/guides/airy-core-in-test-env#connect-sources) guide or the ::: -### Configure the webook integration +### Configure the webhook integration For Facebook to start sending events to your running instance, it must first verify your integration with a challenge. To verify your Facebook webhook @@ -86,36 +86,87 @@ On the `User or Page` option, select `Get Page Token` and click on `Generate Acc You're now ready to connect a Facebook page to the Airy Core Platform 🎉. -## Connect a channel +## Connect Connects a Facebook page to the Airy Core Platform. ``` -POST /channels.connect +POST /facebook.connect ``` -- `source` _must_ be `facebook` -- `source_channel_id` is the Facebook page id -- `token` is the page Access Token +- `page_id` is the Facebook page id +- `page_token` is the page Access Token +- `name` Custom name for the connected page +- `image_url` Custom image URL + +**Sample request** ```json5 { - "source": "facebook", - "source_channel_id": "fb-page-id-1", - "token": "authentication token", - "name": "My custom name for this page", // optional + "page_id": "fb-page-id-1", + "page_token": "authentication token", + "name": "My custom name for this page", "image_url": "https://example.org/custom-image.jpg" // optional } ``` -**Sample Response** +**Sample response** ```json5 { "id": "channel-uuid-1", "name": "My custom name for this page", - "image_url": "https://example.org/custom-image.jpg", // optional + "image_url": "https://example.org/custom-image.jpg", "source": "facebook", "source_channel_id": "fb-page-id-1" } ``` + +## Disconnect + +Disconnects a Facebook page from the Airy Core Platform + +``` +POST /facebook.disconnect +``` + +import ChannelDisconnect from './channel-disconnect.mdx' + + + +## Explore + +`POST /facebook.explore` + +A synchronous endpoint that makes a request to Facebook +to list the available Facebook pages. Some of those pages may already +be connected, which is accounted for in the boolean field `connected`. Due to +the nature of the request, the response time may vary. + +**Sample request** + +```json5 +{ + "auth_token": "authentication token" +} +``` + +**Sample response** + +```json5 +{ + "data": [ + { + "name": "my page 1", + "page_id": "fb-page-id-1", + "connected": false, + "image_url": "http://example.org/avatar.jpeg" // optional + }, + { + "name": "my page 2", + "page_id": "fb-page-id-2", + "connected": true + } + ] +} +``` diff --git a/docs/docs/sources/google.md b/docs/docs/sources/google.md index 41beb246db..bb29a5715b 100644 --- a/docs/docs/sources/google.md +++ b/docs/docs/sources/google.md @@ -29,28 +29,27 @@ against your partner key. You must also set the environment variable Once the verification process has been completed, Google will immediately start sending events to your Airy Core Platform instance. -## Connect a channel +## Connect Connects a Google Business Account to the Airy Core Platform. ``` -POST /channels.connect +POST /google.connect ``` -- `source` _must_ be `google` -- `source_channel_id` The id of your Google Business Message [agent](https://developers.google.com/business-communications/business-messages/reference/business-communications/rest/v1/brands.agents#Agent). -- `token` leave empty. To allow authentication you must provide a [Google service account key file](https://developers.google.com/business-communications/business-messages/guides/quickstarts/prerequisite-setup) in your runtime configuration. +- `gbm_id` The id of your Google Business Message [agent](https://developers.google.com/business-communications/business-messages/reference/business-communications/rest/v1/brands.agents#Agent). +- `name` Custom name for the connected business +- `image_url` Custom image URL ```json5 { - "source": "google", - "source_channel_id": "gbm-id", + "gbm_id": "gbm-id", "name": "My custom name for this location", "image_url": "https://example.com/custom-image.jpg" // optional } ``` -**Sample Response** +**Sample response** ```json5 { @@ -61,3 +60,13 @@ POST /channels.connect "source_channel_id": "gbm-id" } ``` + +## Disconnect + +``` +POST /google.disconnect +``` + +import ChannelDisconnect from './channel-disconnect.mdx' + + diff --git a/docs/docs/sources/sms-twilio.md b/docs/docs/sources/sms-twilio.md index 47856546b7..a930c9c39d 100644 --- a/docs/docs/sources/sms-twilio.md +++ b/docs/docs/sources/sms-twilio.md @@ -18,7 +18,7 @@ import TwilioSource from './twilio-source.mdx' -## Connect a channel +## Connect After you created a Twilio phone number you must [point its webhook integration](https://www.twilio.com/docs/sms/tutorials/how-to-receive-and-reply-java#configure-your-webhook-url) @@ -27,26 +27,26 @@ to your running Airy Core Platform instance. Next call the Platform API: ``` -POST /channels.connect +POST /twilio.sms.connect ``` -- `source` _must_ be `twilio.sms` -- `source_channel_id` The phone number as listed in your [Twilio +- `phone_number` The phone number as listed in your [Twilio dashboard](https://www.twilio.com/console/phone-numbers/). It must _not_ contain spaces and must include the country code. +- `name` Custom name for the connected phone number +- `image_url` Custom image URL -**Sample Request** +**Sample request** ```json5 { - "source": "twilio.sms", - "source_channel_id": "+491234567", + "phone_number": "+491234567", "name": "SMS for receipts", "image_url": "https://example.com/custom-image.jpg" // optional } ``` -**Sample Response** +**Sample response** ```json5 { @@ -57,3 +57,13 @@ POST /channels.connect "source_channel_id": "+491234567" } ``` + +## Disconnect + +``` +POST /twilio.sms.disconnect +``` + +import ChannelDisconnect from './channel-disconnect.mdx' + + diff --git a/docs/docs/sources/twilio-source.mdx b/docs/docs/sources/twilio-source.mdx index d90905bffe..addc708a0a 100644 --- a/docs/docs/sources/twilio-source.mdx +++ b/docs/docs/sources/twilio-source.mdx @@ -1,8 +1,8 @@ You must create a [Twilio auth token](https://support.twilio.com/hc/en-us/articles/223136027-Auth-Tokens-and-How-to-Change-Them) -and add it to `infrastructure/airy.conf` together with your account sid: +and add it to `infrastructure/airy.conf` together with your account SID: ``` -TWILIO_AUTH_TOKEN= -TWILIO_SID= +authToken= +accountSid= ``` diff --git a/docs/docs/sources/whatsapp-twilio.md b/docs/docs/sources/whatsapp-twilio.md index 2d63e66085..5fdc57e2e8 100644 --- a/docs/docs/sources/whatsapp-twilio.md +++ b/docs/docs/sources/whatsapp-twilio.md @@ -1,6 +1,6 @@ --- -title: Whatsapp via Twilio -sidebar_label: Whatsapp - Twilio +title: WhatsApp via Twilio +sidebar_label: WhatsApp - Twilio --- The Twilio WhatsApp source provides a channel for sending and receiving WhatsApp @@ -28,27 +28,27 @@ to your Airy Core Platform running instance. Next call the Airy Core Platform API for connecting channels: ``` -POST /channels.connect +POST /twilio.whatsapp.connect ``` -- `source` _must_ be `twilio.whatsapp` -- `source_channel_id` The phone number as listed in your [Twilio +- `phone_number` The phone number as listed in your [Twilio dashboard](https://www.twilio.com/console/phone-numbers/). - It must _not_ have spaces, must include the country - code, and be prefixed by `whatsapp:` + It must _not_ have spaces and must include the country + code. +- `name` Custom name for the connected phone number +- `image_url` Custom image URL -**Sample Request** +**Sample request** ```json5 { - "source": "twilio.whatsapp", - "source_channel_id": "whatsapp:+491234567", + "phone_number": "+491234567", "name": "WhatsApp Marketing", "image_url": "https://example.com/custom-image.jpg" // optional } ``` -**Sample Response** +**Sample response** ```json5 { @@ -59,3 +59,13 @@ POST /channels.connect "source_channel_id": "whatsapp:+491234567" } ``` + +## Disconnect + +``` +POST /twilio.whatsapp.disconnect +``` + +import ChannelDisconnect from './channel-disconnect.mdx' + + diff --git a/frontend/README.md b/frontend/README.md index 40bc51d5d8..87dc722d22 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,32 +1,20 @@ +

+ + Airy Logo + +

+ # Frontend -The `frontend` top-level folder of the Airy Core Platform -[monorepo](https://en.wikipedia.org/wiki/Monorepo) contains the UI related code -of the platform. +The Frontend section of the Airy core platform contains the user interaction(UI) related code. Here is a quick introduction to the frontend projects: -- `components` - - Here you can find the [Airy - Components Library](https://www.npmjs.com/package/@airyhq/components library. We are open-sourcing - (it's a work in progress) the critical components necessary to build a - messaging application. We use this library to power our commercial offering. - -- `showcase` - - This directory contains a small React application that showcases the - components of the [Airy Components - Library](https://www.npmjs.com/package/@airyhq/components), we host the `main` - branch version at [https://components.airy.co](https://components.airy.co). -- `demo` +- `Demo` - This project is a minimum UI implementation of the provided [Airy Core Platform API](https://docs.airy.co/api/http). - Unlike `showcase` it does not use the `npm` version of the `components` library, but instead uses the - local repository version. + The [Demo project](https://github.com/airyhq/airy/tree/develop/frontend/demo) is a minimum UI implementation of the provided [Airy Core Platform API](https://docs.airy.co/api/http). -- `chat_plugin` +- `Chat_Plugin` - This project is implements a UI for the [Chat Plugin Source](https://docs.airy.co/sources/chat-plugin) that can - be installed as Javascript tag on websites. + This project is implements a UI for the [Chat Plugin Source](https://docs.airy.co/sources/chat-plugin) that can be installed as Javascript tag on websites. diff --git a/frontend/assets/airy_demo_login.png b/frontend/assets/airy_demo_login.png new file mode 100644 index 0000000000..7a92c0752b Binary files /dev/null and b/frontend/assets/airy_demo_login.png differ diff --git a/frontend/assets/airy_primary_rgb.svg b/frontend/assets/airy_primary_rgb.svg new file mode 100644 index 0000000000..2d6ca166ca --- /dev/null +++ b/frontend/assets/airy_primary_rgb.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/chat-plugin/src/airyRenderProps/AiryBubble/index.tsx b/frontend/chat-plugin/src/airyRenderProps/AiryBubble/index.tsx index a9832f684f..45fbe5a541 100644 --- a/frontend/chat-plugin/src/airyRenderProps/AiryBubble/index.tsx +++ b/frontend/chat-plugin/src/airyRenderProps/AiryBubble/index.tsx @@ -11,17 +11,17 @@ const AiryBubble = (props: Props) => {
props.toggleHideChat()}> {!props.isChatHidden ? ( - + + fillRule="nonzero"> ) : ( - + diff --git a/frontend/chat-plugin/src/airyRenderProps/AiryHeaderBar/index.tsx b/frontend/chat-plugin/src/airyRenderProps/AiryHeaderBar/index.tsx index 7e3e89e817..fb651bc069 100644 --- a/frontend/chat-plugin/src/airyRenderProps/AiryHeaderBar/index.tsx +++ b/frontend/chat-plugin/src/airyRenderProps/AiryHeaderBar/index.tsx @@ -13,12 +13,12 @@ const AiryHeaderBar = (props: Props) => {
diff --git a/frontend/chat-plugin/src/components/chat/index.tsx b/frontend/chat-plugin/src/components/chat/index.tsx index b3bdb3593b..f9498665d1 100644 --- a/frontend/chat-plugin/src/components/chat/index.tsx +++ b/frontend/chat-plugin/src/components/chat/index.tsx @@ -126,6 +126,7 @@ const Chat = (props: Props) => { {messages.map(message => { return ( props.airyMessageProp(ctrl) diff --git a/frontend/chat-plugin/src/components/websocket/index.ts b/frontend/chat-plugin/src/components/websocket/index.ts index bf032c1dee..bf7a41bc80 100644 --- a/frontend/chat-plugin/src/components/websocket/index.ts +++ b/frontend/chat-plugin/src/components/websocket/index.ts @@ -1,10 +1,10 @@ import {Client, messageCallbackType, IFrame} from '@stomp/stompjs'; import 'regenerator-runtime/runtime'; -// @ts-ignore // Default to hostname set by local environment +declare const window: any; const API_HOST = window.airy.h || 'chatplugin.api'; -// @ts-ignore + // Allow turning off ssl (unsafe!) for local development const TLS_PREFIX = window.airy.no_tls === true ? '' : 's'; diff --git a/frontend/chat-plugin/src/defaultScript.tsx b/frontend/chat-plugin/src/defaultScript.tsx index 9d457e2849..dbafc928fe 100644 --- a/frontend/chat-plugin/src/defaultScript.tsx +++ b/frontend/chat-plugin/src/defaultScript.tsx @@ -22,7 +22,8 @@ color: #444; body.appendChild(anchor); +declare const window: any; + new AiryWidget({ - // @ts-ignore channel_id: window.airy.cid, }).render(anchor); diff --git a/frontend/chat-plugin/src/iframe.tsx b/frontend/chat-plugin/src/iframe.tsx index 45f68bef7d..3a441fc580 100644 --- a/frontend/chat-plugin/src/iframe.tsx +++ b/frontend/chat-plugin/src/iframe.tsx @@ -1,7 +1,7 @@ import {h, render} from 'preact'; -const renderMethod = () => { - const App = require('./App').default; +const renderMethod = async () => { + const App = (await import('./App')).default; render(, document.getElementById('root')); }; diff --git a/frontend/chat-plugin/src/routes/chat/index.tsx b/frontend/chat-plugin/src/routes/chat/index.tsx index da62cd995a..5f25d38cb8 100644 --- a/frontend/chat-plugin/src/routes/chat/index.tsx +++ b/frontend/chat-plugin/src/routes/chat/index.tsx @@ -1,3 +1,4 @@ +/* eslint-disable react/display-name */ import {h} from 'preact'; import {AuthConfiguration} from '../../config'; import {RoutableProps} from 'preact-router'; diff --git a/frontend/demo/BUILD b/frontend/demo/BUILD index b452f4c152..a54ed7cb52 100644 --- a/frontend/demo/BUILD +++ b/frontend/demo/BUILD @@ -19,6 +19,8 @@ ts_library( "@npm//react", "@npm//react-facebook-login", "@npm//react-router-dom", + "@npm//react-window", + "@npm//react-window-infinite-loader", "@npm//redux", "@npm//redux-starter-kit", "@npm//reselect", diff --git a/frontend/demo/README.md b/frontend/demo/README.md index 859423c7a9..5fe00df59c 100644 --- a/frontend/demo/README.md +++ b/frontend/demo/README.md @@ -1,3 +1,52 @@ -# Airy Demo UI +

+ Airy Login + +

+ + +### Airy Demo UI + +The Airy Demo UI is a minimal user interactive frontend project that showcases the Airy Core Platform API.It enables users to experience the functionalities of our Airy Core Platform. + + +- [Prerequities](#prerequities) +- [Building Airy Demo UI](#building-airy-demo-ui) +- [Installation](#installation) +- [Authentication](#authentication) +- [Endpoints](#endpoints) + +### Prerequisites + +* [Node.js](https://nodejs.org/) version 10 or newer +* [Git](https://www.atlassian.com/git/tutorials/install-git/) for your platform +* [Bazel](https://docs.bazel.build/versions/3.7.0/install.html) for building and testing the app + + +### Building Airy Demo UI + +You can run the Airy Demo UI locally by running the following commands: + +``` +$ git clone https://github.com/airyhq/airy +$ cd airy +$ AIRY_VERSION=beta ./scripts/bootstrap.sh (Takes a few minutes) +``` +When the bootstrap process finishes, open another terminal and run ``` $ ibazel run //frontend/demo:bundle_server ``` +Then open `http://localhost:8080/` in a web browser to access the Airy Demo UI + +### Installation +The bootstrap installation requires [Vagrant](https://www.vagrantup.com/downloads) and [VirtualBox](https://www.virtualbox.org/wiki/Downloads). If they are not +found, the script ```$ AIRY_VERSION=beta ./scripts/bootstrap.sh``` will attempt to install them for you. Check out our [test deployment guide](/docs/docs/guides/airy-core-in-test-env.md) for detailed information. + +### Authentication + +In order to communicate with our API endpoints, you need a valid [JWT](https://jwt.io/) token. To get a valid token you first need to signup using the signup [endpoint](#endpoints) and then login using the login [endpoint](#endpoints). + +### Endpoints + To communicate with our signup endpoint and register your email, open another terminal and type in the terminal ```curl -X POST -H 'Content-Type: application/json' -d '{"first_name": "your_name","last_name": "your_last_name","password": "your_password","email": "your_email@airy.co"}' http://api.airy/users.signup``` + + To sign in, type in the terminal ```token=$(echo $(curl -H 'Content-Type: application/json' -d \"{ \\"email\":\"your_email@airy.co\",\\"password\":\"your_last_name\" \}" api.airy/users.login) | jq -r '.token')``` + +Aside from Curl, [PostMan](https://www.postman.com/downloads/) and other API testing tools could also be used to access the endpoints. + -This app demos a minimal frontend that showcases the Airy Core Platform API diff --git a/frontend/demo/src/App.tsx b/frontend/demo/src/App.tsx index 19f6d9cbbf..9dac7583b1 100644 --- a/frontend/demo/src/App.tsx +++ b/frontend/demo/src/App.tsx @@ -5,15 +5,16 @@ import {withRouter, Route, Switch, Redirect, RouteComponentProps} from 'react-ro import {AiryLoader} from '@airyhq/components'; import TopBar from './components/TopBar'; import Login from './pages/Login'; +import Channels from './pages/Channels'; +import Inbox from './pages/Inbox'; +import Tags from './pages/Tags'; +import Logout from './pages/Logout'; import NotFound from './pages/NotFound'; import Sidebar from './components/Sidebar'; import {StateModel} from './reducers'; -import {CHANNELS_ROUTE, LOGIN_ROUTE, ROOT_ROUTE, TAGS_ROUTE} from './routes/routes'; - -import {Tags} from './pages/Tags'; -import Channels from './pages/Channels'; +import {INBOX_ROUTE, CHANNELS_ROUTE, LOGIN_ROUTE, LOGOUT_ROUTE, ROOT_ROUTE, TAGS_ROUTE} from './routes/routes'; import styles from './App.module.scss'; @@ -31,7 +32,7 @@ class App extends Component & RouteComponentPro return this.props.user.token && this.props.user.token !== ''; } - shouldShowSidebar = (path: string) => { + shouldShowSidebar = () => { return this.isAuthSuccess; }; @@ -47,7 +48,7 @@ class App extends Component & RouteComponentPro return (
- {this.shouldShowSidebar(this.props.location.pathname) ? ( + {this.shouldShowSidebar() ? ( <> @@ -57,10 +58,12 @@ class App extends Component & RouteComponentPro )} - {this.isAuthSuccess ? : } + {this.isAuthSuccess ? : } + + diff --git a/frontend/demo/src/actions/conversations/index.ts b/frontend/demo/src/actions/conversations/index.ts new file mode 100644 index 0000000000..9e0c987a1f --- /dev/null +++ b/frontend/demo/src/actions/conversations/index.ts @@ -0,0 +1,73 @@ +import {Dispatch} from 'redux'; +import {createAction} from 'typesafe-actions'; +import {doFetchFromBackend} from '../../api/airyConfig'; + +import {Conversation, ConversationPayload, conversationsMapper} from '../../model/Conversation'; +import {ResponseMetadata} from '../../model/ResponseMetadata'; +import {StateModel} from '../../reducers'; + +export const CONVERSATION_LOADING = '@@conversation/LOADING'; +export const CONVERSATIONS_LOADING = '@@conversations/LOADING'; +export const CONVERSATIONS_MERGE = '@@conversations/MERGE'; +export const CONVERSATION_ADD_ERROR = '@@conversations/ADD_ERROR_TO_CONVERSATION'; +export const CONVERSATION_REMOVE_ERROR = '@@conversations/REMOVE_ERROR_FROM_CONVERSATION'; + +export const loadingConversationAction = createAction(CONVERSATION_LOADING, resolve => (conversationId: string) => + resolve(conversationId) +); + +export const loadingConversationsAction = createAction(CONVERSATIONS_LOADING, resolve => () => resolve()); + +export const mergeConversationsAction = createAction( + CONVERSATIONS_MERGE, + resolve => (conversations: Conversation[], responseMetadata: ResponseMetadata) => + resolve({conversations, responseMetadata}) +); + +export const addErrorToConversationAction = createAction( + CONVERSATION_ADD_ERROR, + resolve => (conversationId: string, errorMessage: string) => resolve({conversationId, errorMessage}) +); + +export const removeErrorFromConversationAction = createAction( + CONVERSATION_REMOVE_ERROR, + resolve => (conversationId: string) => resolve({conversationId}) +); + +export interface FetchConversationsResponse { + data: ConversationPayload[]; + metadata: ResponseMetadata; +} + +export function fetchConversations() { + return async (dispatch: Dispatch) => { + dispatch(loadingConversationsAction()); + return doFetchFromBackend('conversations.list', { + page_size: 10, + }) + .then((response: FetchConversationsResponse) => { + dispatch(mergeConversationsAction(conversationsMapper(response.data), response.metadata)); + return Promise.resolve(true); + }) + .catch((error: Error) => { + return Promise.reject(error); + }); + }; +} + +export function fetchNextConversations() { + return async (dispatch: Dispatch, state: StateModel) => { + const cursor = state.data.conversations.all.metadata.next_cursor; + dispatch(loadingConversationsAction()); + return doFetchFromBackend('conversations.list', { + cursor, + }) + .then((response: FetchConversationsResponse) => { + dispatch(mergeConversationsAction(conversationsMapper(response.data), response.metadata)); + return Promise.resolve(true); + }) + .catch((error: Error) => { + return Promise.reject(error); + }); + }; +} diff --git a/frontend/demo/src/actions/tags/index.tsx b/frontend/demo/src/actions/tags/index.tsx index 663c62111b..7dcc776232 100644 --- a/frontend/demo/src/actions/tags/index.tsx +++ b/frontend/demo/src/actions/tags/index.tsx @@ -18,7 +18,7 @@ export const deleteTagAction = createAction(DELETE_TAG, resolve => (id: string) export const filterTagAction = createAction(SET_TAG_FILTER, resolve => (filter: string) => resolve(filter)); export const errorTagAction = createAction(ERROR_TAG, resolve => (status: string) => resolve(status)); -export function getTags(query: string = '') { +export function getTags() { return function(dispatch: Dispatch) { return doFetchFromBackend('tags.list').then((response: GetTagsResponse) => { dispatch(fetchTagAction(tagsMapper(response.data))); @@ -51,7 +51,7 @@ export function updateTag(tag: Tag) { id: tag.id, name: tag.name, color: tag.color, - }).then((responseTag: Tag) => dispatch(editTagAction(tag))); + }).then(() => dispatch(editTagAction(tag))); }; } @@ -71,7 +71,7 @@ export function filterTags(filter: string) { }; } -export function errorTag(status: string, data?: string) { +export function errorTag(status: string) { return function(dispatch: Dispatch) { dispatch(errorTagAction(status)); }; diff --git a/frontend/demo/src/actions/user/index.ts b/frontend/demo/src/actions/user/index.ts index 64639649b2..85c87cb5ae 100644 --- a/frontend/demo/src/actions/user/index.ts +++ b/frontend/demo/src/actions/user/index.ts @@ -10,29 +10,19 @@ const USER_AUTH_ERROR = '@@auth/ERROR'; const USER_LOGOUT = '@@auth/LOGOUT_USER'; export const setCurrentUserAction = createAction(SET_CURRENT_USER, resolve => (user: User) => resolve(user)); - export const userAuthErrorAction = createAction(USER_AUTH_ERROR, resolve => (error: Error) => resolve(error)); - export const logoutUserAction = createAction(USER_LOGOUT); - -export const logoutUser = () => { - return function(dispatch: Dispatch) { - dispatch(logoutUserAction()); - }; -}; - export interface LoginViaEmailRequestPayload { - email: String; - password: String; + email: string; + password: string; } export function loginViaEmail(requestPayload: LoginViaEmailRequestPayload) { return async (dispatch: Dispatch) => { return doFetchFromBackend('users.login', requestPayload) .then((response: UserPayload) => { - const user = userMapper(response); - dispatch(setCurrentUserAction(user)); - return Promise.resolve(user); + dispatch(setCurrentUserAction(userMapper(response))); + return Promise.resolve(true); }) .catch((error: Error) => { dispatch(userAuthErrorAction(error)); @@ -40,3 +30,9 @@ export function loginViaEmail(requestPayload: LoginViaEmailRequestPayload) { }); }; } + +export function logoutUser() { + return async (dispatch: Dispatch) => { + dispatch(logoutUserAction()); + }; +} diff --git a/frontend/demo/src/api/cookie.ts b/frontend/demo/src/api/cookie.ts index 9ee8b80761..f30660d56f 100644 --- a/frontend/demo/src/api/cookie.ts +++ b/frontend/demo/src/api/cookie.ts @@ -1,4 +1,4 @@ -export function setCookie(name: string, value: string, domain: string, days: number = 3650) { +export function setCookie(name: string, value: string, domain: string, days = 3650) { let expires: string; if (days) { const date = new Date(); diff --git a/frontend/demo/src/assets/images/icons/airy-icon.svg b/frontend/demo/src/assets/images/icons/airy-icon.svg new file mode 100644 index 0000000000..671c694b3b --- /dev/null +++ b/frontend/demo/src/assets/images/icons/airy-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/demo/src/assets/images/icons/airy_avatar.svg b/frontend/demo/src/assets/images/icons/airy_avatar.svg new file mode 100644 index 0000000000..6ccf9258a9 --- /dev/null +++ b/frontend/demo/src/assets/images/icons/airy_avatar.svg @@ -0,0 +1,18 @@ + + + avatar/Airy Live Agent + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/frontend/demo/src/assets/images/icons/arrow-left-2.svg b/frontend/demo/src/assets/images/icons/arrow-left-2.svg new file mode 100644 index 0000000000..3b70e9c287 --- /dev/null +++ b/frontend/demo/src/assets/images/icons/arrow-left-2.svg @@ -0,0 +1,11 @@ + + + + icon/arrow/arrow-left copy + Created with Sketch. + + + + + + \ No newline at end of file diff --git a/frontend/demo/src/assets/images/icons/facebook_rounded.svg b/frontend/demo/src/assets/images/icons/facebook_rounded.svg new file mode 100644 index 0000000000..34cc7ba1ad --- /dev/null +++ b/frontend/demo/src/assets/images/icons/facebook_rounded.svg @@ -0,0 +1,20 @@ + + + + Group 3 + Created with Sketch. + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/frontend/demo/src/assets/images/icons/google-messages.svg b/frontend/demo/src/assets/images/icons/google-messages.svg new file mode 100644 index 0000000000..922380f68b --- /dev/null +++ b/frontend/demo/src/assets/images/icons/google-messages.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/demo/src/assets/images/icons/google_avatar.svg b/frontend/demo/src/assets/images/icons/google_avatar.svg new file mode 100644 index 0000000000..68b33c0898 --- /dev/null +++ b/frontend/demo/src/assets/images/icons/google_avatar.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/demo/src/assets/images/icons/inbox.svg b/frontend/demo/src/assets/images/icons/inbox.svg new file mode 100644 index 0000000000..048285f515 --- /dev/null +++ b/frontend/demo/src/assets/images/icons/inbox.svg @@ -0,0 +1,13 @@ + + + + + + + + diff --git a/frontend/demo/src/assets/images/icons/messenger_avatar.svg b/frontend/demo/src/assets/images/icons/messenger_avatar.svg new file mode 100644 index 0000000000..7276cb745c --- /dev/null +++ b/frontend/demo/src/assets/images/icons/messenger_avatar.svg @@ -0,0 +1,14 @@ + + + avatar/Messenger + + + + + + + + + + + \ No newline at end of file diff --git a/frontend/demo/src/assets/images/icons/search.svg b/frontend/demo/src/assets/images/icons/search.svg new file mode 100644 index 0000000000..2ae349b29b --- /dev/null +++ b/frontend/demo/src/assets/images/icons/search.svg @@ -0,0 +1,13 @@ + + + + Icon / Search / Static + Created with Sketch. + + + + + + + + diff --git a/frontend/demo/src/assets/images/icons/sms-icon.svg b/frontend/demo/src/assets/images/icons/sms-icon.svg new file mode 100644 index 0000000000..11428838ae --- /dev/null +++ b/frontend/demo/src/assets/images/icons/sms-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/demo/src/assets/images/icons/sms_avatar.svg b/frontend/demo/src/assets/images/icons/sms_avatar.svg new file mode 100644 index 0000000000..c7de6dc3f8 --- /dev/null +++ b/frontend/demo/src/assets/images/icons/sms_avatar.svg @@ -0,0 +1,12 @@ + + + Group 16 + + + + + + + + + \ No newline at end of file diff --git a/frontend/demo/src/assets/images/icons/whatsapp-icon.svg b/frontend/demo/src/assets/images/icons/whatsapp-icon.svg new file mode 100644 index 0000000000..f6f002d96b --- /dev/null +++ b/frontend/demo/src/assets/images/icons/whatsapp-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/demo/src/assets/images/icons/whatsapp_avatar.svg b/frontend/demo/src/assets/images/icons/whatsapp_avatar.svg new file mode 100644 index 0000000000..95773d4399 --- /dev/null +++ b/frontend/demo/src/assets/images/icons/whatsapp_avatar.svg @@ -0,0 +1,10 @@ + + + Group 14 + + + + + + + \ No newline at end of file diff --git a/frontend/demo/src/components/ColorSelector.tsx b/frontend/demo/src/components/ColorSelector.tsx index 47e78b8b95..553eee8aee 100644 --- a/frontend/demo/src/components/ColorSelector.tsx +++ b/frontend/demo/src/components/ColorSelector.tsx @@ -1,4 +1,4 @@ -import React, {useCallback, useState} from 'react'; +import React, {useCallback} from 'react'; import {connect} from 'react-redux'; import {RootState} from '../reducers'; import {TagSettings} from '../model/Tag'; diff --git a/frontend/demo/src/components/Dialog.tsx b/frontend/demo/src/components/Dialog.tsx index 01c1792f95..52f6e78ed7 100644 --- a/frontend/demo/src/components/Dialog.tsx +++ b/frontend/demo/src/components/Dialog.tsx @@ -14,7 +14,7 @@ type DialogProps = { close: () => void; }; -const Dialog = ({children, close, style, coverStyle, overlay}: DialogProps) => { +const Dialog: React.FC = ({children, close, style, coverStyle, overlay}: DialogProps): JSX.Element => { const keyDown = useCallback( (e: KeyboardEvent) => { if (e.key === 'Escape') { diff --git a/frontend/demo/src/components/IconChannel/index.tsx b/frontend/demo/src/components/IconChannel/index.tsx new file mode 100644 index 0000000000..cf23b68799 --- /dev/null +++ b/frontend/demo/src/components/IconChannel/index.tsx @@ -0,0 +1,150 @@ +/* eslint-disable react/display-name */ +import React from 'react'; +import {Channel} from '../../model/Channel'; + +import {ReactComponent as FacebookIcon} from '../../assets/images/icons/facebook_rounded.svg'; +import {ReactComponent as GoogleIcon} from '../../assets/images/icons/google-messages.svg'; +import {ReactComponent as SmsIcon} from '../../assets/images/icons/sms-icon.svg'; +import {ReactComponent as WhatsappIcon} from '../../assets/images/icons/whatsapp-icon.svg'; +import {ReactComponent as MessengerAvatar} from '../../assets/images/icons/messenger_avatar.svg'; +import {ReactComponent as GoogleAvatar} from '../../assets/images/icons/google_avatar.svg'; +import {ReactComponent as SmsAvatar} from '../../assets/images/icons/sms_avatar.svg'; +import {ReactComponent as WhatsappAvatar} from '../../assets/images/icons/whatsapp_avatar.svg'; +import {ReactComponent as AiryAvatar} from '../../assets/images/icons/airy_avatar.svg'; +import {ReactComponent as AiryIcon} from '../../assets/images/icons/airy-icon.svg'; + +import styles from './style.module.scss'; + +type IconChannelProps = { + channel: Channel; + icon?: boolean; + avatar?: boolean; + name?: boolean; + text?: boolean; +}; + +const PlaceholderChannelData = { + id: 'id', + name: 'Retriving Data...', + source: 'FACEBOOK', + sourceChannelId: 'external_channel_id', + connected: true, +}; + +const IconChannel: React.FC = ({ + channel, + icon, + avatar, + name, + text, +}: IconChannelProps): JSX.Element => { + if (!channel) { + channel = PlaceholderChannelData; + } + + const SOURCE_INFO = { + facebook: { + text: 'Facebook page', + icon: function() { + return ; + }, + avatar: function() { + return ; + }, + name: channel.name, + }, + google: { + text: 'Google page', + icon: function() { + return ; + }, + avatar: function() { + return ; + }, + name: channel.name, + }, + 'twilio.sms': { + text: 'SMS page', + icon: function() { + return ; + }, + avatar: function() { + return ; + }, + name: channel.name, + }, + 'twilio.whatsapp': { + text: 'Whatsapp page', + icon: function() { + return ; + }, + avatar: function() { + return ; + }, + name: channel.name, + }, + chat_plugin: { + text: 'Airy Chat plugin', + icon: function() { + return ; + }, + avatar: function() { + return ; + }, + name: channel.name, + }, + }; + + //TODO: This has to go once the backend returns the source + const channelInfo = SOURCE_INFO[channel.source || 'chat_plugin']; + const fbFallback = SOURCE_INFO['FACEBOOK']; + + if (!channelInfo) { + return ( + <> + {fbFallback.icon()} {fbFallback.text} + + ); + } + + if (icon && name) { + return ( +
+ {channelInfo.icon()} +

{channelInfo.name}

+
+ ); + } else if (avatar && name) { + return ( +
+ {channelInfo.avatar()} +

{channelInfo.name}

+
+ ); + } else if (icon && text) { + return ( +
+ {channelInfo.icon()} +

{channelInfo.text}

+
+ ); + } else if (avatar && text) { + return ( +
+ {channelInfo.avatar()} +

{channelInfo.text}

+
+ ); + } else if (icon) { + return <>{channelInfo.icon()}; + } else if (avatar) { + return <>{channelInfo.avatar()}; + } + return ( + <> + {fbFallback.icon()} {fbFallback.text} + + ); +}; + +export default IconChannel; diff --git a/frontend/demo/src/components/IconChannel/style.module.scss b/frontend/demo/src/components/IconChannel/style.module.scss new file mode 100644 index 0000000000..f69c552777 --- /dev/null +++ b/frontend/demo/src/components/IconChannel/style.module.scss @@ -0,0 +1,45 @@ +.iconName { + display: flex; + align-items: center; + min-width: 0px; + div { + height: 20px; + width: 20px; + } + svg { + height: 20px; + width: 20px; + } + p { + overflow: hidden; + text-overflow: ellipsis; + } +} + +.avatarName { + display: flex; + align-items: center; + min-width: 0px; + div { + height: 20px; + width: 20px; + } + svg { + height: 20px; + width: 20px; + } + p { + overflow: hidden; + text-overflow: ellipsis; + } +} + +.iconText { + display: flex; + align-items: center; +} + +.avatarText { + display: flex; + align-items: center; +} diff --git a/frontend/demo/src/components/ListenOutsideClick/index.tsx b/frontend/demo/src/components/ListenOutsideClick/index.tsx index 1b0c3d8958..f41eb8ba92 100644 --- a/frontend/demo/src/components/ListenOutsideClick/index.tsx +++ b/frontend/demo/src/components/ListenOutsideClick/index.tsx @@ -6,7 +6,11 @@ type ListenOutsideClickProps = { onOuterClick: (event: React.MouseEvent) => void; }; -const ListenOutsideClick = ({children, className, onOuterClick}: ListenOutsideClickProps) => { +const ListenOutsideClick: React.FC = ({ + children, + className, + onOuterClick, +}: ListenOutsideClickProps): JSX.Element => { const innerRef = useRef(null); useEffect(() => { diff --git a/frontend/demo/src/components/ResizableWindowList/index.module.scss b/frontend/demo/src/components/ResizableWindowList/index.module.scss new file mode 100644 index 0000000000..81086b9878 --- /dev/null +++ b/frontend/demo/src/components/ResizableWindowList/index.module.scss @@ -0,0 +1,3 @@ +.resizablewindowlist { + height: 100%; +} diff --git a/frontend/demo/src/components/ResizableWindowList/index.tsx b/frontend/demo/src/components/ResizableWindowList/index.tsx new file mode 100644 index 0000000000..de1d3633c3 --- /dev/null +++ b/frontend/demo/src/components/ResizableWindowList/index.tsx @@ -0,0 +1,79 @@ +import React, {Component} from 'react'; +import {ComponentType} from 'react'; +import {ListOnItemsRenderedProps, ListChildComponentProps, VariableSizeList as List} from 'react-window'; +import './index.module.scss'; + +type ResizableWindowProps = { + itemCount: number; + itemSize?: number | ((index: number) => number); + width: string; + children: ComponentType; + onItemsRendered: (event: ListOnItemsRenderedProps) => void; +}; + +type ResizableWindowState = { + height: number; + mounted: boolean; +}; + +class ResizableWindowList extends Component { + resizeRef: React.RefObject; + listRef: React.MutableRefObject; + + constructor(props: ResizableWindowProps) { + super(props); + this.resizeRef = React.createRef(); + this.listRef = React.createRef(); + this.state = {height: 250, mounted: true}; + } + + componentDidMount() { + window.addEventListener('resize', this.resizeToFit); + this.resizeToFit(); + setTimeout(this.resizeAfterStart, 500); + } + + resizeAfterStart = () => { + this.resizeRef.current ? this.resizeToFit() : setTimeout(this.resizeAfterStart, 500); + }; + + componentWillUnmount() { + this.setState({mounted: false}); + window.removeEventListener('resize', this.resizeToFit); + } + + resizeToFit = () => { + if (this.state.mounted) { + this.setState({ + height: Math.floor(this.resizeRef.current.getBoundingClientRect().height) + 200, + }); + } + }; + + resetSizeCache = () => { + this.listRef.current && this.listRef.current.resetAfterIndex(0, true); + }; + + setRef = (ref: List) => { + this.listRef.current = ref; + }; + + render() { + const {itemCount, itemSize, width, children, onItemsRendered} = this.props; + return ( +
+ (typeof itemSize === 'function' ? itemSize(item) : itemSize)} + width={width}> + {children} + +
+ ); + } +} + +export default ResizableWindowList; diff --git a/frontend/demo/src/components/SearchField.tsx b/frontend/demo/src/components/SearchField.tsx index a3e65de008..d9a65645ef 100644 --- a/frontend/demo/src/components/SearchField.tsx +++ b/frontend/demo/src/components/SearchField.tsx @@ -13,7 +13,14 @@ type SearchFieldProps = { autoFocus?: boolean; }; -export const SearchField = ({id, placeholder, value, setValue, resetClicked, autoFocus}: SearchFieldProps) => { +export const SearchField: React.FC = ({ + id, + placeholder, + value, + setValue, + resetClicked, + autoFocus, +}: SearchFieldProps): JSX.Element => { const inputRef = createRef(); const resetButton = useCallback(() => { setValue(''); diff --git a/frontend/demo/src/components/Sidebar/index.tsx b/frontend/demo/src/components/Sidebar/index.tsx index d4ef67d170..924224c9bd 100644 --- a/frontend/demo/src/components/Sidebar/index.tsx +++ b/frontend/demo/src/components/Sidebar/index.tsx @@ -2,9 +2,10 @@ import React from 'react'; import {withRouter, Link, matchPath, RouteProps} from 'react-router-dom'; import {ReactComponent as PlugIcon} from '../../assets/images/icons/git-merge.svg'; +import {ReactComponent as InboxIcon} from '../../assets/images/icons/inbox.svg'; import {ReactComponent as TagIcon} from '../../assets/images/icons/price-tag.svg'; -import {CHANNELS_ROUTE, TAGS_ROUTE} from '../../routes/routes'; +import {INBOX_ROUTE, CHANNELS_ROUTE, TAGS_ROUTE} from '../../routes/routes'; import styles from './index.module.scss'; @@ -16,6 +17,12 @@ const Sidebar = (props: RouteProps) => { return (