Skip to content

Commit a87d962

Browse files
shiveshrshiveshr
authored
Merge branch 'master' into json-compatibility
2 parents 68619b1 + 0c06ff6 commit a87d962

File tree

27 files changed

+3273
-329
lines changed

27 files changed

+3273
-329
lines changed

‎.travis.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
language: java
1111
install: true
1212
jdk:
13-
- openjdk8
13+
- oraclejdk11
1414

1515
env:
1616
global:

‎charts/schema-registry/Chart.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,6 @@ apiVersion: v2
1212
name: schema-registry
1313
description: Schema Registry Helm chart for Kubernetes
1414
type: application
15-
version: 0.0.1
16-
appVersion: 0.0.1
15+
version: 0.1.0
16+
appVersion: 0.1.0
1717
home: https://github.com/pravega/schema-registry

‎charts/schema-registry/README.md

+4-2
Original file line numberDiff line numberDiff line change
@@ -49,14 +49,16 @@ The following table lists the configurable parameters of the schema registry cha
4949
| Parameter | Description | Default |
5050
| ----- | ----------- | ------ |
5151
| `replicas` | Number of replicas | `2` |
52-
| `image.repository` | Repository for schema registry image | `pravega/schema-registry` |
53-
| `image.tag` | Tag for schema registry image | `0.0.1` |
52+
| `image.repository` | Repository for schema registry image | `pravega/schemaregistry` |
53+
| `image.tag` | Tag for schema registry image | `0.1.0` |
5454
| `image.pullPolicy` | Pull policy for schema registry image | `IfNotPresent` |
5555
| `serviceAccount.create` | Whether to create a service account | `true` |
5656
| `serviceAccount.name` | Name of the service account | `schema-registry` |
5757
| `serviceAccount.annotations` | Annotations to be added to the service account | |
5858
| `service.type` | Schema registry service type | `LoadBalancer` |
5959
| `service.port` | Schema registry service port | `9092` |
60+
| `initContainer` | Configuration for the init container | `{}` |
61+
| `container` | Additional configuration for the schema registry container, to be provided if an init container has been configured | `{}` |
6062
| `ingress.enabled` | Whether to expose as an ingress resource | `false` |
6163
| `ingress.path` | Path for the ingress | `/` |
6264
| `ingress.annotations` | Annotations for the ingress | `{}` |

‎charts/schema-registry/templates/configmap.yaml

+8-1
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,17 @@ data:
3232
"DISABLE_BASIC_AUTH": "{{ .Values.authentication.disableBasicAuthentication }}"
3333
{{- end }}
3434
{{- if .Values.authentication.authorizationResourceQualifier }}
35-
"AUTHORIZATION_RESOURCE_QUALIFIER": "{{ .Values.authentication.authorizationResourceQualifier }}"
35+
"AUTHORIZATION_DOMAIN_RESOURCE_IDENTIFIER": "{{ .Values.authentication.authResourceDomain }}"
36+
"AUTHORIZATION_NAMESPACE_RESOURCE_IDENTIFIER": "{{ .Values.authentication.authResourceNamespace }}"
37+
"AUTHORIZATION_GROUP_RESOURCE_IDENTIFIER": "{{ .Values.authentication.authorizationResourceGroup }}"
3638
{{- end }}
3739
{{- if .Values.tls.enabled }}
3840
"TLS_CERT_FILE": "/etc/secret-volume/{{ .Values.tls.certFile }}"
3941
"TLS_KEY_FILE": "/etc/secret-volume/{{ .Values.tls.keyFile }}"
4042
"TLS_KEY_PASSWORD_FILE": "/etc/secret-volume/{{ .Values.tls.keyPasswordFile }}"
4143
{{- end }}
44+
{{- if and .Values.container .Values.container.command }}
45+
"PRE_START_CMD": "{{ .Values.container.command }}"
46+
{{- else }}
47+
"PRE_START_CMD": ""
48+
{{- end }}

‎charts/schema-registry/templates/deployment.yaml

+27-2
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ spec:
4646
- sh
4747
- -c
4848
- netstat -ltn 2> /dev/null | grep {{ .Values.service.port }} || ss -ltn 2> /dev/null | grep {{ .Values.service.port }}
49-
{{- if or .Values.authentication.enabled .Values.tls.enabled }}
49+
{{- if or .Values.authentication.enabled .Values.tls.enabled (and .Values.container .Values.container.volume) }}
5050
volumeMounts:
5151
{{- if .Values.authentication.enabled }}
5252
- name: auth-passwd-secret
@@ -58,8 +58,29 @@ spec:
5858
mountPath: /etc/secret-volume
5959
readOnly: true
6060
{{- end }}
61+
{{- if and .Values.container .Values.container.volume }}
62+
- name: transfer-data
63+
mountPath: {{ .Values.container.volume.mountPath }}
6164
{{- end }}
62-
{{- if or .Values.authentication.enabled .Values.tls.enabled }}
65+
{{- end }}
66+
{{- if .Values.initContainer }}
67+
initContainers:
68+
- name: {{ .Values.initContainer.name }}
69+
image: "{{ .Values.initContainer.image.repository }}:{{ .Values.initContainer.image.tag }}"
70+
imagePullPolicy: {{ .Values.initContainer.image.pullPolicy }}
71+
command:
72+
{{ toYaml .Values.initContainer.command | indent 12 }}
73+
{{- if .Values.initContainer.resources }}
74+
resources:
75+
{{ toYaml .Values.initContainer.command | indent 12 }}
76+
{{- end }}
77+
{{- if .Values.initContainer.volume }}
78+
volumeMounts:
79+
- name: transfer-data
80+
mountPath: {{ .Values.initContainer.volume.mountPath }}
81+
{{- end }}
82+
{{- end }}
83+
{{- if or .Values.authentication.enabled .Values.tls.enabled (and .Values.initContainer .Values.initContainer.volume) }}
6384
volumes:
6485
{{- if .Values.authentication.enabled }}
6586
- name: auth-passwd-secret
@@ -71,4 +92,8 @@ spec:
7192
secret:
7293
secretName: {{ .Values.tls.secret }}
7394
{{- end }}
95+
{{- if and .Values.container .Values.container.volume }}
96+
- name: transfer-data
97+
emptyDir: {}
98+
{{- end }}
7499
{{- end }}

‎charts/schema-registry/values.yaml

+25-3
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
replicas: 2
1616

1717
image:
18-
repository: pravega/schema-registry
19-
tag: 0.0.1
18+
repository: pravega/schemaregistry
19+
tag: 0.1.0
2020
pullPolicy: IfNotPresent
2121

2222
## Service account name and whether to create it.
@@ -29,6 +29,26 @@ service:
2929
type: LoadBalancer
3030
port: 9092
3131

32+
initContainer: {}
33+
# name: ""
34+
# image:
35+
# repository:
36+
# tag:
37+
# pullPolicy: IfNotPresent
38+
## provide a list containing the command that needs to be executed within the init container
39+
## e.g. ["/bin/sh", "-c", "cp /data/* /target"]
40+
# command: []
41+
# resources: {}
42+
# volume:
43+
# mountPath: /target
44+
45+
container: {}
46+
## just provide the command to be executed within the entrypoint
47+
## e.g. "cp /additional_libs/* /opt/schema-registry/lib/"
48+
# command: ""
49+
# volume:
50+
# mountPath: /additional_libs
51+
3252
## Ingress configuration.
3353
ingress:
3454
enabled: false
@@ -67,8 +87,10 @@ authentication:
6787
## authentication is enabled
6888
passwordAuthSecret:
6989
userPasswordFile:
70-
authorizationResourceQualifier: ""
7190
disableBasicAuthentication: false
91+
authResourceDomain: "prn"
92+
authNamespaceResourceIdentifier: "namespace"
93+
authGroupResourceIdentifier: "group"
7294

7395
tls:
7496
enabled: false

‎docker/pravega-schema-registry/scripts/entrypoint.sh

+4
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,8 @@ if [ ${WAIT_FOR} ];then
2121
${SCRIPTS_DIR}/wait_for
2222
fi
2323

24+
if [[ -n "$PRE_START_CMD" ]];then
25+
$PRE_START_CMD
26+
fi
27+
2428
exec /opt/schema-registry/bin/schema-registry

‎gradle.properties

+2-2
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ gradleGitPluginVersion=2.2.0
4747
avroVersion=1.9.1
4848
avroProtobufVersion=1.7.7
4949
snappyVersion=1.1.7.3
50-
pravegaVersion=0.8.0-2623.279ac21-SNAPSHOT
51-
pravegaKeyCloakVersion=0.7.0
50+
pravegaVersion=0.9.0-2664.5410d55-SNAPSHOT
51+
pravegaKeyCloakVersion=0.8.0
5252

5353
# Version and base tags can be overridden at build time
5454
schemaregistryVersion=0.2.0-SNAPSHOT

‎serializers/avro/src/main/java/io/pravega/schemaregistry/serializer/avro/impl/AvroDeserializer.java

+32-20
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@
99
*/
1010
package io.pravega.schemaregistry.serializer.avro.impl;
1111

12-
import com.google.common.base.Charsets;
12+
import com.google.common.annotations.VisibleForTesting;
1313
import com.google.common.base.Preconditions;
14+
import com.google.common.collect.ImmutableMap;
1415
import io.pravega.schemaregistry.serializer.avro.schemas.AvroSchema;
1516
import io.pravega.schemaregistry.client.SchemaRegistryClient;
1617
import io.pravega.schemaregistry.contract.data.SchemaInfo;
@@ -19,48 +20,59 @@
1920
import io.pravega.schemaregistry.serializer.shared.impl.SerializerConfig;
2021
import org.apache.avro.Schema;
2122
import org.apache.avro.io.BinaryDecoder;
23+
import org.apache.avro.io.DatumReader;
2224
import org.apache.avro.io.DecoderFactory;
2325
import org.apache.avro.reflect.ReflectDatumReader;
2426
import org.apache.avro.specific.SpecificDatumReader;
2527
import org.apache.avro.specific.SpecificRecordBase;
2628

2729
import java.io.IOException;
2830
import java.io.InputStream;
31+
import java.nio.ByteBuffer;
2932
import java.util.concurrent.ConcurrentHashMap;
3033

3134
class AvroDeserializer<T> extends AbstractDeserializer<T> {
32-
private final AvroSchema<T> avroSchema;
33-
private final ConcurrentHashMap<SchemaInfo, Schema> knownSchemas;
35+
private final ConcurrentHashMap<ByteBuffer, DatumReader<T>> knownSchemaReaders;
36+
private final boolean specific;
37+
private final Schema readerSchema;
3438

3539
AvroDeserializer(String groupId, SchemaRegistryClient client,
3640
AvroSchema<T> schema,
3741
SerializerConfig.Decoders decoder, EncodingCache encodingCache) {
3842
super(groupId, client, schema, false, decoder, encodingCache, true);
3943
Preconditions.checkNotNull(schema);
40-
this.avroSchema = schema;
41-
this.knownSchemas = new ConcurrentHashMap<>();
44+
this.knownSchemaReaders = new ConcurrentHashMap<>();
45+
specific = SpecificRecordBase.class.isAssignableFrom(schema.getTClass());
46+
readerSchema = schema.getSchema();
47+
ByteBuffer schemaData = schema.getSchemaInfo().getSchemaData();
48+
knownSchemaReaders.put(schemaData, createDatumReader(readerSchema, readerSchema, specific));
4249
}
4350

4451
@Override
4552
public final T deserialize(InputStream inputStream, SchemaInfo writerSchemaInfo, SchemaInfo readerSchemaInfo) throws IOException {
4653
Preconditions.checkNotNull(writerSchemaInfo);
47-
Schema writerSchema;
48-
if (knownSchemas.containsKey(writerSchemaInfo)) {
49-
writerSchema = knownSchemas.get(writerSchemaInfo);
50-
} else {
51-
String schemaString = new String(writerSchemaInfo.getSchemaData().array(), Charsets.UTF_8);
52-
writerSchema = new Schema.Parser().parse(schemaString);
53-
knownSchemas.put(writerSchemaInfo, writerSchema);
54-
}
55-
Schema readerSchema = avroSchema.getSchema();
54+
final ByteBuffer writerSchemaData = writerSchemaInfo.getSchemaData();
55+
DatumReader<T> datumReader = knownSchemaReaders.computeIfAbsent(writerSchemaData, key -> {
56+
Schema writerSchema = AvroSchema.from(writerSchemaInfo).getSchema();
57+
return createDatumReader(writerSchema, this.readerSchema, specific);
58+
});
5659
BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null);
57-
58-
if (SpecificRecordBase.class.isAssignableFrom(avroSchema.getTClass())) {
59-
SpecificDatumReader<T> datumReader = new SpecificDatumReader<>(writerSchema, readerSchema);
60-
return datumReader.read(null, decoder);
60+
return datumReader.read(null, decoder);
61+
}
62+
63+
@VisibleForTesting
64+
DatumReader<T> createDatumReader(Schema writerSchema, Schema readerSchema, boolean specific) {
65+
DatumReader<T> datumReader;
66+
if (specific) {
67+
datumReader = new SpecificDatumReader<>(writerSchema, readerSchema);
6168
} else {
62-
ReflectDatumReader<T> datumReader = new ReflectDatumReader<>(writerSchema, readerSchema);
63-
return datumReader.read(null, decoder);
69+
datumReader = new ReflectDatumReader<>(writerSchema, readerSchema);
6470
}
71+
return datumReader;
72+
}
73+
74+
@VisibleForTesting
75+
ImmutableMap<ByteBuffer, DatumReader<T>> getKnownSchemaReaders() {
76+
return ImmutableMap.copyOf(knownSchemaReaders);
6577
}
6678
}

‎serializers/avro/src/main/java/io/pravega/schemaregistry/serializer/avro/impl/AvroGenericDeserializer.java

+17-9
Original file line numberDiff line numberDiff line change
@@ -9,41 +9,49 @@
99
*/
1010
package io.pravega.schemaregistry.serializer.avro.impl;
1111

12+
import com.google.common.annotations.VisibleForTesting;
1213
import com.google.common.base.Preconditions;
13-
import io.pravega.schemaregistry.serializer.avro.schemas.AvroSchema;
14+
import com.google.common.collect.ImmutableMap;
1415
import io.pravega.schemaregistry.client.SchemaRegistryClient;
1516
import io.pravega.schemaregistry.contract.data.SchemaInfo;
17+
import io.pravega.schemaregistry.serializer.avro.schemas.AvroSchema;
1618
import io.pravega.schemaregistry.serializer.shared.impl.AbstractDeserializer;
1719
import io.pravega.schemaregistry.serializer.shared.impl.EncodingCache;
1820
import io.pravega.schemaregistry.serializer.shared.impl.SerializerConfig;
1921
import org.apache.avro.Schema;
2022
import org.apache.avro.generic.GenericDatumReader;
2123
import org.apache.avro.io.BinaryDecoder;
2224
import org.apache.avro.io.DecoderFactory;
23-
25+
import org.apache.commons.lang3.tuple.Pair;
2426
import javax.annotation.Nullable;
2527
import java.io.IOException;
2628
import java.io.InputStream;
2729
import java.util.concurrent.ConcurrentHashMap;
2830

2931
public class AvroGenericDeserializer extends AbstractDeserializer<Object> {
30-
private final ConcurrentHashMap<SchemaInfo, Schema> knownSchemas;
32+
private final ConcurrentHashMap<Pair<SchemaInfo, SchemaInfo>, GenericDatumReader<Object>> knownSchemaReaders;
3133

3234
public AvroGenericDeserializer(String groupId, SchemaRegistryClient client, @Nullable AvroSchema<Object> schema,
3335
SerializerConfig.Decoders decoder, EncodingCache encodingCache) {
3436
super(groupId, client, schema, false, decoder, encodingCache, true);
35-
this.knownSchemas = new ConcurrentHashMap<>();
37+
this.knownSchemaReaders = new ConcurrentHashMap<>();
3638
}
3739

3840
@Override
3941
public final Object deserialize(InputStream inputStream, SchemaInfo writerSchemaInfo, SchemaInfo readerSchemaInfo) throws IOException {
4042
Preconditions.checkNotNull(writerSchemaInfo);
41-
Schema writerSchema = knownSchemas.computeIfAbsent(writerSchemaInfo, x -> AvroSchema.from(x).getSchema());
42-
Schema readerSchema = knownSchemas.computeIfAbsent(readerSchemaInfo, x -> AvroSchema.from(x).getSchema());
43-
44-
GenericDatumReader<Object> genericDatumReader = new GenericDatumReader<>(writerSchema, readerSchema);
45-
43+
final Pair<SchemaInfo, SchemaInfo> keyPair = Pair.of(writerSchemaInfo, readerSchemaInfo);
44+
GenericDatumReader<Object> genericDatumReader = knownSchemaReaders.computeIfAbsent(keyPair, key -> {
45+
Schema writerSchema = AvroSchema.from(writerSchemaInfo).getSchema();
46+
Schema readerSchema = AvroSchema.from(readerSchemaInfo).getSchema();
47+
return new GenericDatumReader<>(writerSchema, readerSchema);
48+
});
4649
BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null);
4750
return genericDatumReader.read(null, decoder);
4851
}
52+
53+
@VisibleForTesting
54+
ImmutableMap<Pair<SchemaInfo, SchemaInfo>, GenericDatumReader<Object>> getKnownSchemaReaders() {
55+
return ImmutableMap.copyOf(knownSchemaReaders);
56+
}
4957
}

‎serializers/avro/src/main/java/io/pravega/schemaregistry/serializer/avro/impl/AvroSerializer.java

+16-16
Original file line numberDiff line numberDiff line change
@@ -27,33 +27,33 @@
2727
import java.io.OutputStream;
2828

2929
public class AvroSerializer<T> extends AbstractSerializer<T> {
30-
private final AvroSchema<T> avroSchema;
30+
31+
private final SpecificDatumWriter<T> specificDatumWriter;
32+
private final GenericDatumWriter<T> genericDatumWriter;
33+
private final ReflectDatumWriter<T> reflectDatumWriter;
34+
3135
public AvroSerializer(String groupId, SchemaRegistryClient client, AvroSchema<T> schema,
32-
Encoder encoder, boolean registerSchema) {
36+
Encoder encoder, boolean registerSchema) {
3337
super(groupId, client, schema, encoder, registerSchema, true);
34-
this.avroSchema = schema;
38+
Schema avroSchema = schema.getSchema();
39+
this.specificDatumWriter = new SpecificDatumWriter<>(avroSchema);
40+
this.genericDatumWriter = new GenericDatumWriter<>(avroSchema);
41+
this.reflectDatumWriter = new ReflectDatumWriter<>(avroSchema);
3542
}
3643

3744
@Override
3845
protected void serialize(T var, SchemaInfo schemaInfo, OutputStream outputStream) throws IOException {
39-
Schema schema = avroSchema.getSchema();
40-
4146
BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null);
4247

43-
if (IndexedRecord.class.isAssignableFrom(var.getClass())) {
44-
if (SpecificRecord.class.isAssignableFrom(var.getClass())) {
45-
SpecificDatumWriter<T> writer = new SpecificDatumWriter<>(schema);
46-
writer.write(var, encoder);
47-
} else {
48-
GenericDatumWriter<T> writer = new GenericDatumWriter<>(schema);
49-
writer.write(var, encoder);
50-
}
48+
if (SpecificRecord.class.isAssignableFrom(var.getClass())) {
49+
specificDatumWriter.write(var, encoder);
50+
} else if (IndexedRecord.class.isAssignableFrom(var.getClass())) {
51+
genericDatumWriter.write(var, encoder);
5152
} else {
52-
ReflectDatumWriter<T> writer = new ReflectDatumWriter<>(schema);
53-
writer.write(var, encoder);
53+
reflectDatumWriter.write(var, encoder);
5454
}
5555

5656
encoder.flush();
5757
outputStream.flush();
5858
}
59-
}
59+
}

0 commit comments

Comments
 (0)