Added MIT_COMPLIANT flag.

catalog-refresh fixes

docker/.env.example

@@ -13,6 +13,15 @@ JITSU_PUBLIC_URL=
 
 # Connectors specific
 SYNCS_ENABLED=false
+# Source Connectors License
+# Jitsu is licensed under MIT license, but some source connectors may use other licenses.
+# If you base your product on Jitsu and want to stay under MIT licese, set this variable to true
+# All not MIT-compatible connector sources will be disabled
+# You can enable all source connectors by setting this variable to false
+# All source connectors will be enabled, but particular connectors license may conflict with license of your product
+MIT_COMPLIANT=true
+
+
 # bulker host how it is reachable from k8s cluster
 EXTERNAL_BULKER_HOST=
 #postgres host how it is reachable from k8s cluster

docker/docker-compose.yml

@@ -126,6 +126,7 @@ services:
       ROTOR_URL: "http://rotor:3401"
       BULKER_URL: "http://bulker:3042"
       BULKER_AUTH_KEY: ${BULKER_PASSWORD}
+      MIT_COMPLIANT: ${MIT_COMPLIANT:-true}
       DATABASE_URL: "postgresql://postgres:${POSTGRES_PASSWORD:-default}@postgres:5432/postgres"
       REDIS_URL: "redis://default:${REDIS_PASSWORD:-default}@redis:6379"
       TEST_CREDENTIALS: ${TEST_CREDENTIALS}

libs/core-functions/src/meta.ts

@@ -50,7 +50,7 @@ export const MixpanelCredentials = z.object({
   simplifiedIdMerge: z
     .boolean()
     .optional()
-    .default(true)
+    .default(false)
     .describe(
       `Simplified Identity Merge::Use Mixpanel <a href="https://docs.mixpanel.com/docs/tracking-methods/identifying-users">Simplified Identity Merge</a> feature.<br/>Enable this option if your Mixpanel project has the corresponding <a href="https://docs.mixpanel.com/docs/tracking-methods/identifying-users#how-do-i-switch-between-the-simplified-and-original-api">feature enabled</a>.<br/><b>Using this feature is highly recommended to achieve better quality Identity Merge</b>`
     ),

webapps/console/components/ServiceEditor/ServiceEditor.tsx

@@ -40,9 +40,10 @@ export const ServiceEditor: React.FC<ServiceEditorProps> = props => {
   const { object, meta, createNew, onCancel, onDelete, onTest, isNew, noun, loadMeta } = props;
   const appConfig = useAppConfig();
   const workspace = useWorkspace();
-  const { push } = useRouter();
+  const { push, query } = useRouter();
   const [obj, setObj] = useState<Partial<ServiceConfig>>({
     ...props.object,
+    version: query.version ? query.version.toString() : props.object?.version,
   });
   const [credentials, setCredentials] = useState<any>(obj.credentials || {});
   const [isTouched, setIsTouched] = useState<boolean>(false);
@@ -298,7 +299,11 @@ export const ServiceEditor: React.FC<ServiceEditorProps> = props => {
         <EditorField
           key={"version"}
           id={"version"}
-          help={`Version of package: ${obj.package || meta.packageId}`}
+          help={`Version of package: ${obj.package || meta.packageId} License: ${
+            Array.isArray(meta.meta.mitVersions) && meta.meta.mitVersions.includes(obj.version)
+              ? "MIT"
+              : meta.meta.license
+          }`}
           label={"Version"}
           required={true}
         >

webapps/console/components/ServicesCatalog/ServicesCatalog.tsx

@@ -8,6 +8,7 @@ import { LoadingAnimation } from "../GlobalLoader/GlobalLoader";
 import React from "react";
 import { ErrorCard } from "../GlobalError/GlobalError";
 import { Input } from "antd";
+import { useAppConfig } from "../../lib/context";
 
 function groupByType(sources: SourceType[]): Record<string, SourceType[]> {
   const groups: Record<string, SourceType[]> = {};
@@ -43,7 +44,7 @@ function groupByType(sources: SourceType[]): Record<string, SourceType[]> {
 export function getServiceIcon(source: SourceType, icons: Record<string, string> = {}): React.ReactNode {
   const connectorSubtype = source.meta.connectorSubtype;
 
-  const logoSvg = source.logoSvg || icons[source.packageId];
+  const logoSvg = (source.logoSvg || icons[source.packageId]) as string;
   return logoSvg ? (
     <img src={"data:image/svg+xml;base64," + Buffer.from(logoSvg).toString("base64")} alt={source.meta.name} />
   ) : connectorSubtype === "database" ? (
@@ -57,6 +58,7 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
   const { data, isLoading, error } = useApi<{ sources: SourceType[] }>(`/api/sources?mode=meta`);
   const sourcesIconsLoader = useApi<{ sources: SourceType[] }>(`/api/sources?mode=icons-only`);
   const [filter, setFilter] = React.useState("");
+  const appconfig = useAppConfig();
   const sourcesIcons: Record<string, string> = sourcesIconsLoader.data
     ? sourcesIconsLoader.data.sources.reduce(
         (acc, item) => ({
@@ -92,6 +94,7 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
             .filter(source => source.meta.name && source.meta.name.toLowerCase().includes(filter.toLowerCase()))
             .filter(
               source =>
+                !appconfig.mitCompliant ||
                 source.meta.license?.toLowerCase() === "mit" ||
                 (source.meta.mitVersions && source.meta.mitVersions.length > 0)
             );

webapps/console/lib/schema/index.ts

@@ -84,6 +84,7 @@ export const AppConfig = z.object({
       provider: z.enum(["google-cloud-scheduler"]).optional(),
     }),
   }),
+  mitCompliant: z.boolean().optional(),
   nango: z
     .object({
       publicKey: z.string(),

webapps/console/pages/[workspaceId]/services.tsx

@@ -154,7 +154,9 @@ const ServicesList: React.FC<{}> = () => {
       const rawVersions = await rpc(
         `/api/sources/versions?type=${packageType}&package=${encodeURIComponent(packageId)}`
       );
-      const versions = rawVersions.versions.filter((v: any) => v.isRelease && v.isMit).map((v: any) => v.name);
+      const versions = rawVersions.versions
+        .filter((v: any) => v.isRelease && (v.isMit || !appconfig.mitCompliant))
+        .map((v: any) => v.name);
       const sourceType = await rpc(`/api/sources/${packageType}/${encodeURIComponent(packageId)}`);
 
       return {

webapps/console/pages/api/admin/catalog-refresh.ts

@@ -75,37 +75,45 @@ async function process(src: string) {
   const res = await fetch(metadataUrl);
   let packageId: string | undefined = undefined;
   let metadata: any = {};
-  let icon: string | undefined = undefined;
+  let icon: Buffer | undefined = undefined;
+  log.atInfo().log(`Processing ${src}: ${metadataUrl}`);
   if (res.ok) {
-    metadata = yaml.load(await res.text());
+    metadata = yaml.load(await res.text(), { json: true });
 
     const license = metadata.data?.license?.toLowerCase();
 
-    if (license === "elv2") {
-      const commitHistory = `https://api.github.com/repos/${repo}/commits?path=/${basePath}/${src}/metadata.yaml&per_page=100`;
-      log.atWarn().log(`Source ${src} has ELv2 license. Looking for MIT versions at ${commitHistory}`);
-      const commits = await rpc(commitHistory);
-      log.atInfo().log(`Found ${commits.length} commits`);
-      for (const { sha } of commits) {
-        const commitFile = `https://raw.githubusercontent.com/${repo}/${sha}/${basePath}/${src}/metadata.yaml`;
-        const oldYml = await fetch(commitFile);
-        if (oldYml.ok) {
-          const oldMeta = yaml.load(await oldYml.text());
-          const license = oldMeta.data?.license?.toLowerCase() || "unknown-license";
-          if (license === "mit") {
-            const dockerVersion = oldMeta.data?.dockerImageTag;
-            if (!dockerVersion) {
-              log.atWarn().log(`MIT version of ${src} doesn't have dockerImageTag: ${commitFile}`);
+    if (!license || license.toLowerCase() !== "mit") {
+      const pageSize = 100; // max supported by github API
+      const commitHistory = `https://api.github.com/repos/${repo}/commits?path=/${basePath}/${src}/metadata.yaml&per_page=${pageSize}`;
+      log.atWarn().log(`Source ${src} has ${license} license. Looking for MIT versions at ${commitHistory}`);
+      for (let page = 1; page <= 10; page++) {
+        const commits = await rpc(commitHistory + "&page=" + page);
+        log.atInfo().log(`Source ${src} found ${commits.length} commits (page ${page})`);
+        for (const { sha } of commits) {
+          const commitFile = `https://raw.githubusercontent.com/${repo}/${sha}/${basePath}/${src}/metadata.yaml`;
+          const oldYml = await fetch(commitFile);
+          if (oldYml.ok) {
+            const oldMeta = yaml.load(await oldYml.text(), { json: true });
+            const license = oldMeta.data?.license?.toLowerCase() || "unknown-license";
+            if (license === "mit") {
+              const dockerVersion = oldMeta.data?.dockerImageTag;
+              if (!dockerVersion) {
+                log.atWarn().log(`MIT version of ${src} doesn't have dockerImageTag: ${commitFile}`);
+              } else {
+                log.atInfo().log(`Found MIT version of ${src} --> ${dockerVersion}`);
+                mitVersions.add(dockerVersion);
+              }
             } else {
-              log.atWarn().log(`Found MIT version of ${src} --> ${dockerVersion}`);
-              mitVersions.add(dockerVersion);
+              otherVersions[license] = otherVersions[license] || new Set<string>();
+              otherVersions[license].add(license);
             }
           } else {
-            otherVersions[license] = otherVersions[license] || new Set<string>();
-            otherVersions[license].add(license);
+            log.atWarn().log(`Failed to load ${commitFile}`);
           }
-        } else {
-          log.atWarn().log(`Failed to load ${commitFile}`);
+        }
+        if (commits.length < pageSize) {
+          // no more commits
+          break;
         }
       }
     }
@@ -120,7 +128,7 @@ async function process(src: string) {
     const iconUrl = `https://raw.githubusercontent.com/${repo}/master/${basePath}/${src}/icon.svg`;
     const iconRes = await fetch(iconUrl);
     if (iconRes.ok) {
-      icon = await iconRes.text();
+      icon = Buffer.from(await iconRes.arrayBuffer());
     } else {
       log.atWarn().log(`Source ${src} icon file ${metadata.data?.icon} doesn't exist at ${iconUrl}`);
     }
@@ -144,7 +152,7 @@ async function process(src: string) {
     await db.prisma().connectorPackage.update({ where: { id: currentId }, data });
     return { [`${packageId}`]: "updated" };
   } else {
-    log.atInfo().log(`Created ${packageId} info. Has icon: ${!!icon}, has metadata: ${!!metadata}`);
+    log.atInfo().log(`Creating ${packageId} info. Has icon: ${!!icon}, has metadata: ${!!metadata}`);
     await db.prisma().connectorPackage.create({ data: data });
     return { [`${packageId}`]: "created" };
   }

webapps/console/pages/api/app-config.ts

@@ -56,6 +56,7 @@ export default createRoute()
             host: nangoConfig.nangoApiHost,
           }
         : undefined,
+      mitCompliant: isTruish(process.env.MIT_COMPLIANT),
     };
   })
   .toNextApiHandler();

webapps/console/prisma/schema.prisma

@@ -77,6 +77,7 @@ model Workspace {
   featuresEnabled         String[] @default([])
 
   @@unique(slug)
+  @@index(updatedAt(sort: Desc))
 }
 
 model WorkspaceAccess {
@@ -135,6 +136,8 @@ model ConfigurationObject {
 
   fromLinks ConfigurationObjectLink[] @relation("from")
   toLinks   ConfigurationObjectLink[] @relation("to")
+
+  @@index(updatedAt(sort: Desc))
 }
 
 model ConfigurationObjectLink {
@@ -153,14 +156,16 @@ model ConfigurationObjectLink {
 
   toId String
   to   ConfigurationObject @relation(fields: [toId], references: [id], name: "from")
+
+  @@index(updatedAt(sort: Desc))
 }
 
 model ConnectorPackage {
   id        String   @id @default(cuid())
   packageId String
   packageType String @default("airbyte")
   meta      Json?
-  logoSvg   String?
+  logoSvg   Bytes?
   createdAt DateTime @default(now()) /// @zod.custom(z.coerce.date())
   updatedAt DateTime @default(now()) @updatedAt /// @zod.custom(z.coerce.date())
 }