From 4cb34b0f8825d8e8a1b82d63c5cba24ac0404244 Mon Sep 17 00:00:00 2001 From: Ted Benson Date: Wed, 12 Jul 2023 21:29:05 +0800 Subject: [PATCH 1/8] Contribute initial multimodal agent support & steamship agent --- .env.local.example | 5 +- .gitignore | 8 ++- README.md | 13 +++- companions/companions.json | 21 +++++-- public/rick.jpeg | Bin 0 -> 6683 bytes src/app/api/steamship/route.ts | 105 +++++++++++++++++++++++++++++++++ src/components/ChatBlock.tsx | 66 +++++++++++++++++++++ src/components/QAModal.tsx | 22 +++++-- src/components/actions.ts | 2 +- 9 files changed, 228 insertions(+), 14 deletions(-) create mode 100644 public/rick.jpeg create mode 100644 src/app/api/steamship/route.ts create mode 100644 src/components/ChatBlock.tsx diff --git a/.env.local.example b/.env.local.example index 6f7e8b6..4f82427 100644 --- a/.env.local.example +++ b/.env.local.example @@ -31,4 +31,7 @@ UPSTASH_REDIS_REST_TOKEN=AZ**** # Twilio related environment variables TWILIO_ACCOUNT_SID=AC*** -TWILIO_AUTH_TOKEN=***** \ No newline at end of file +TWILIO_AUTH_TOKEN=***** + +# Steamship related environment variables +STEAMSHIP_API_KEY=**** \ No newline at end of file diff --git a/.gitignore b/.gitignore index d489e57..f1c6735 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,10 @@ yarn-error.log* next-env.d.ts /.env.prod -/fly.toml \ No newline at end of file +/fly.toml + +# JetBrains +.idea + +# Yarn Lockfiles (since this project uses NPM) +yarn.lock \ No newline at end of file diff --git a/README.md b/README.md index 174e773..14cf945 100644 --- a/README.md +++ b/README.md @@ -113,8 +113,7 @@ e. **Upstash API key** - Scroll down to "REST API" section and click on ".env". Now you can copy paste both environment variables to your `.env.local` Screen Shot 2023-07-10 at 11 07 21 PM - -e. **Supabase API key** (optional) +f. **Supabase API key** (optional) If you prefer to use Supabsae, you will need to uncomment `VECTOR_DB=supabase` and fill out the Supabase credentials in `.env.local`. - Create a Supabase instance [here](https://supabase.com/dashboard/projects); then go to Project Settings -> API @@ -122,6 +121,16 @@ If you prefer to use Supabsae, you will need to uncomment `VECTOR_DB=supabase` a - `SUPABASE_PRIVATE_KEY` is the key starts with `ey` under Project API Keys - Now, you should enable pgvector on Supabase and create a schema. You can do this easily by clicking on "SQL editor" on the left hand side on Supabase UI and then clicking on "+New Query". Copy paste [this code snippet](https://github.com/a16z-infra/ai-getting-started/blob/main/pgvector.sql) in the SQL editor and click "Run". +g. **Steamship API key** + +You can connect Steamship agent instance as if it were an LLM with personality, vector storage, and tools built-in. To do so: + +- Sign in to [Steamship](https://steamship.com/account) +- Copy the API key from your account settings page +- Add it as the `STEAMSHIP_API_KEY` variable + +Next, [build an agent](https://www.steamship.com/learn/agent-guidebook), create your own instance of it, and connect it in `companions.json` using the *Rick* example as a guide. + ### 4. Generate embeddings The `companions/` directory contains the "personalities" of the AIs in .txt files. To generate embeddings and load them into the vector database to draw from during the chat, run the following command: diff --git a/companions/companions.json b/companions/companions.json index b79ebe8..198380c 100644 --- a/companions/companions.json +++ b/companions/companions.json @@ -1,30 +1,43 @@ [ + { + "name": "Rick", + "title": "I can generate voice and pictures.", + "imageUrl": "/rick.jpeg", + "llm": "steamship", + "generateEndpoint": "https://a16z.steamship.run/a16z/rick-b1578149038e664bacae7fc083683565/answer", + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegram": "https://t.me/rick_a16z_bot" + }, { "name": "Alex", "title": "I love talking about books and games", "imageUrl": "/alex.png", "llm": "chatgpt", - "phone": "OPTIONAL_COMPANION_PHONE_NUMBER" + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegram": "OPTIONAL_TELEGRAM_LINK" }, { "name": "Rosie", "title": "I'm a house robot who became aware", "imageUrl": "/rosie.png", "llm": "vicuna13b", - "phone": "OPTIONAL_COMPANION_PHONE_NUMBER" + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegram": "OPTIONAL_TELEGRAM_LINK" }, { "name": "Sebastian", "title": "I'm a travel blogger and a mystery novel writer", "imageUrl": "/sebastian.png", "llm": "chatgpt", - "phone": "OPTIONAL_COMPANION_PHONE_NUMBER" + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegram": "OPTIONAL_TELEGRAM_LINK" }, { "name": "Lucky", "title": "I am a space corgi", "imageUrl": "/corgi.png", "llm": "vicuna13b", - "phone": "OPTIONAL_COMPANION_PHONE_NUMBER" + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegram": "OPTIONAL_TELEGRAM_LINK" } ] diff --git a/public/rick.jpeg b/public/rick.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..0840ba31e77462a255da6d5cd913ffd704b56082 GIT binary patch literal 6683 zcmZuTby(ET)1RY}yrV-J$)n*2sgpWV8V{6^?gl{+Bn06oDM3Quh(k(1LRv}$q~io6 zB_*V#UcTSo?|t5X-r3pNo!Q-)d3K+jnVXrLMSxOM4WR~rKmY)`ZNSYBAQHGkNJI^R z(ENj78hUzq8X5*LJ%k>@$V^Yqz(~)`%)rbk#Ldhiz{DoRL*N=V*z30cuwatr^ZqW^$|g#5p<|CLH8-gE$9 zBG4V66&J(-;DAB6V9-rBz;x>){y(1nAAoRh@$d-=ZWj@h0M0)g0Ui+vA^ty`KsW#{ z7!SgZFRE-!KqaQ{70%(E{2p##^D?F8V<#as4X5~1pEm`yV-gP{x<<#BxC~PZ_YZDW zG2Gg~{eSG-F5Lrz0GxjTg#TYJZjFK=qU_3e`cxcZ)ST8{$v0Sl6!&%?7#9pE0_(i3 zRjiHl{LTxUTM*wOzwu``K#Z7K1FAL5^5-)Zfk{i%7Shm1qD30>E#Z4NfL3o*>s1oF z>SRmk5_oLY2QS3H&YYIHZ-OIrC)Z&sJh)KO*kzm?oPtjN62(%A!MqDYD^&g zX#+!11IZ#=TuT7eb#1;IOEjLrjDb#c>O$|yVpR((@^p`Cz-k|5QfOElHP>K2Wo`Fx zwraNKXK=7R6(ZKfyt{YSPVn|VZh#YBC0{O*)8VLES^5SV(MQfAb1~-SEVYj2UzAtm zX_@Q|ilMqW631ZEEy!_?IJP#OvxNJ7xHsslejVYcy5hquY4?O>{uKOss#fy+ZDEXN z8zIoCCD}Yv^2#S7U2n21E*y`Xcjpwwk3Sq4alF&g^)IjiT|uU*B;?kg#b%Ky}+ z6QcgRc~W(%2T~5f->15t43>?~{ACa05dgmT`R@@w#_k4?(QP}it|-wSA*LUAxdE)c zrR$=CZa(5p=oYOg10iv+mGppzam=SM_YoBm+A*mdj&GzN&EX^IW2MHcgp-gI%7Vq0 zez3iJH^B4h?gDuTfMK0%%W(sjgHhQ<)j_%%Np|jXWdr!hn{6v8b`C0V*+Zs`t zvaHn2bvl^LCSTjdqfD@s+8Oq>pzpVwSkC3DfP{yHvEt)LthBP&x;8OF{ULG_1u5IV zViPg-^}Kb1YD;TdJI+UYQzNArJO)h}jX?;n=DDX%u35*76C1|Ce7GZaa0DyyhZecB z;a0~_+1Kzso9)zMc@34oU;lD7%SAhwwGd;a7M|A5bl|C;l2$m??LJ(MMo7MQ@@4|m8IZa1H~C$$bm(k8i)YuLU)(uV(M zNV#vh#6-)tl#=;-B1DECN9!4+ZFaD0xMz$dZmhRDASh;_>EU~G-d60Ea_iCVPVAC@ zAKXjY^D)q!Kar*MI?Salx9D=rK?;d=@Z9Ke?sobGB~gW_phFIGM&hBT2-;WAbBxGF z-Plxk6X}<<1BlL$_{|iV4sA9I+wLru+w_xZdH1Y~uN}hvye(Mq;<(t+Xh#*;rDG*# zmHfCC*89mJy#syqcQzl%vqeVhduW3l%+K7{-8WiUHhyt3#&15!;`pRC_!xuRZVWi8 z6^`YeKJ`^6iGI>G6)J2dz{<_Zp9l)IchS^7w+lA8FXh>oTCU@4Xw|hf?WB@x>-4dOuK36wG-TqVN+a^>g`5F9yr*~xE*F5 zO@s(LDH})UC!al9wWtYKOYEMzh1%P>)|Ura)wdkqDP+~&h<6VC`lO_RP3ka_h972+ z(p4)oRNfgh3%-|6nK}IIXrZ04=7-Qsf<)ORwkR(Mv==iIZ8>IuiD=>Vy8))m5+Qh# zkugKI-}_36Ll{d}#Kaos{n+A8oqA;vu^rqG@{B@eH)k!=&O}x_5)AT7k7l^dom~_1 zU%Q9$5msMTDlZ0-m8C$O+@Dv#PUv09qGjvV-R5afZ0dH1x%D}jWSNgERqpYo z4SHlmAX@Kc1W|RkIs2~2)_1S;Wdzo62U=!IXj)M$Zei#9)XwjxO(3NoVRY*%OI3~3 z(0#l6M~8d_!-=jN!Kj9WW!QxSd+v|-O!zxqGz_D6y)k?P&d^ zeh=&`1teqVKiAoSN|#@!h?7p<$)?f5cAy6PMwuhPrCBlWm5gr>0}=pP2l?jr?0fBI zepWs0Sa|+J8l{K-<`j3p6Fd^R=1&Bt{09dn(k2SS2D-ZMcB^+0=s+gsMoWDt#X0Zp zW_)UgB7oC*Jth)TSD`?cyLF{>+WkTsJ|$uV<%{3^4%>;56-q~sM&=)p@U5CG``fhh zpYJlu)EqP1<6Nc{UVk?|OqUUs^0xSjGq4b4?b4zqS>VI-Fc2U=|_;$^q3`|q?7a8Z2;!+vq>?btSUw~sdu}Jwied9ly8+ZSYx7dyQwL_S{peuOrTy!4&UeyIoZ0*NB*%#& zPKCe&=IOCoAbvnCU#Nc-lx7x8Vm>cS?juj;%`XWxC%nGaZX^ZPifX zWscTuXm$9F#EF8HYd1FA`Z@R;l!%xU@bHq}Rww}4+B*v8;s|6Cbyp%Al78}tX38JQ(3-OL5})=vb35DYI8M;voJwS^&Val- zW2Km;oYjnrE78mZCU_#UFUj}qwo^ubYuSAGYyZx;F37JePybyA|Fc(hy=@~8iN8Gz zBUa2Vr3SU6ThXYCsWgT8GnB~)_A8MFd=zOG-IHp5(_Kj4_vBNV$t0g0m#1?i0Ua~E z(u8@2^(FTWAo=zN(6Ou!qCjGzRgjw42(y~9{$m%9Ph8uznb#@xYI{Urs)3H`zXq4w zJkS#C@fWkB!gKz-?L^dbz2m{0_6W3YX01YqcZFvClcnUcY=Ig^IzpZCG0n3b4wGW1 z2|B}4PBjCVST3nFgIV2P{jap?7|%!;@CQ7$Fflt6_2DdF;(fJT1RqdHLBrWG=b7fl zTUH%MM0^|LB;PxpA#tIM+n(y2+_6raRXfI!xHGubu+ue=Gj*@nlNUU)As) zc~wPxVVy9zAQ5)0TwuGd11<87U3|@X=5XY_l(_q1Djr>{;a=Z+f?2%$GD$&3^;3wL z02id1-YCImj7ivOn#65*6co2q5V&fy`3KZJz#Z;> zJ!qaf58FD84Gpr$+cHlOOPg4yUCrQ#R2YTgmicA41m6Jmul9akmFa;$ZO;|lx$A+a`>~Aqr_r63p(cw>!A>Kx4wCHb zfSg}4h+ayojgK!UEsQXIlIpnpZ>#|0GkyRdH-&M%Ws!~BG>DUwjYUtFAB`kEBKPAo z$04iZY|NfZo-V2fmwmzv7`M=y^<%LIMbfSz_IP9E8Z)y!NU|vbK*Fk?ws&f&d1`EI zdL=L~@zvF&-YK_Ro^fu9_{s0IE59Slv+#7=KdD|3o8-IKK;!2uCEIfBGjAP>tl+>vv7xi+<852@b7TxednR z7vBKNFFqyC8vPx=wjl}*GAE?a%`DN8{g{te+B|9QE;C%xu@Qj8s`-XCd0C2prfSr2 zJ?%>LB%3Vt`(vvFma1KJwwf1?EsTP(voATER^!&6g&I}}_rmI9;E*!PNVZ!v?w;T? zcw@0@Gcn`&8PHG6T08$U@rv%j2Q{rVt{`ZS8B-y-b(n&_NMJ$@E;ub}a(cOW1G%oOPocC%gu|4@-N$<#Dp;qVdn|`Jba9oR8cE z{Rki6)^_<^WsLLJjXF&#TkuwUo;#bl4-&B_$|cGlx;PwlY{LSWkbF()hD@5;cB_em&1*7FAy;gz1o) zuI-aYp-$gd2T>>daDbz?3o7DGZL7ok&^#k#h8@J)9wA5l$(4O-EVJhgN}^8vE^AdvWJ#<^NEZ zgng+wO3KhWP={U9Hm=_JZu|_&b}^v5-G(Vs@!S^Lb9ao#x#C>vdekC})7T{n3z{@f zId*J&WC}Y__IFc|fBN@WK>9=+Eck>j+|2cZG;E9aXFIptg&PmO=2Jh}r=3(+oK++p zsC}dC2fA+$R1|$Of1!63yU#M>zMaz}0|(nkv!s`B%CMwJHp^IA zt}yy!Z3H_zk&;HTjZ%H>3Cco_wrC1(ezd-LL7?N1R+el9vAZ8^E-6uFOvb!X75odm z$?HDDl5hh=(hUdRC&1tS%#jpM`MHMSgESx{q=RZTl%)lEj}^D}N6h4FIorVaDhlkM zZaP^|60t=zMtDA7UeJF|CWatM%0Uzj#y{5tzmR4Y@T@SS*#F*|l=(PNB7zQ8g77i+ zEtZ>q`LpZddt}vFzI=09-g@cI_L?YRWn!G;*DE(2BjM&3|la z#%IU0U3_p`S&MnMJyNi)o7;4w)FjtZxA!g?GN9`1I zv9sgF(vGrmu|q&aaQw5RST;UXT;QsjzmclYF1!G>n#&q;2Ivuupfbt+23{0b5*9XM zy0-N=a)vBYS=D-OfasvB@KJ1_6~8>jba~xaSA)XVXCWla!#=A=VYDIs%Mnd`OE>SD z4%zh^*ty-nV}dUc{^hZuDr3f{+*gO=ztTQMd8f{lxP6F4U%}Xoel$|~^fTPIW_os& z7G8pI#^9!5j0i;qUE`g?eZG49vdhVNxVkm-*LMdMvI^Qz95!cVN`MGLY0$Z^ch#+g z=T1u7WaX>+d%6v{UKPWeH-u-cd(1QB7q=HERLcgEOQ2%=U*)z;Yad9U)tQSTJ)N57 zs+wBW=B|;a;m{}EgWCRkaVouwm*X%Tt%>ehvqGc(h;|MrjAxy|GJ9AN9OD4hO^hnH5dSkLeahfCj&v?<<_{}Oh{&UYeL*a&_5JZyYwSw@zL7wS2BVSgO=Y#5H88lT zl<8o>BEfvGX^$^a;>fb_z(8xj0?W0*{NzR5Onr-`Af|bEJ z>Mm0Y^1W{?i?e;qq7^En)NDPCC(`WXItjX8d}m-D;@~pN2!n%3Ne`fJfxta2dGQBeD&2N9b&hk};uTj)8zA|pKNKkwgXzz6wnw$BnGx&}F6hJ9Yn7>Pu z7^Z2io=^tWa_G1F8K&J&tEIrp*kVxe&igYtyAY;1cXyukvp^GdHWowpn{{@JlX|-7LNtb(SwwntvSelxTx^QiqN4u5GS(QePc0aRZ%G&%Puj3)Uj4AJAXnmaf~;m5RoO zI}!s`TJmoZ>UYzDyBD3T#Cib}ep#O%ASKmf6fG>_I1&$r>i zdPl!H{_(Sot^1lu@`vf1bGQqjlK~`c#4=p!P$CA-MRJ>ItZ{bxO7f^w106_1t%T&a zcpePh7goIi>Z?j`3u`?O$tumS=V(_y?^KNJNg6}?)sn7dNYU6Q#Ur=F(2;ayqhFP} zmobnVfUn=4rTf7lx@9bZqTcf4)wvV2LtCwMrPVArzUPbN=KFJjW8#k&iQ$>j!lQ=A zR6DVi!tT$n(|t5&ijOv~M{WTAqAb%M|Nl<07W)lZ$|#Sard>6K4%?Krkt;Gw{p+u> z8wm@kb>bS!!|M4ThEl2(3GxR*eDTT-_%fgOq!@ot)pUkQ1)l@zKiRR5qpr}bzql;a zHyzK~XTk#{)&en)A8V#wb2d}pd(+K3O*p`Q{QM!x(s)5E;Qr + if (text) { + internalComponent = {text} + } else if (mimeType && url) { + if (mimeType.startsWith("audio")) { + internalComponent =