Skip to content

Commit b4a281e

Browse files
hangtersaopstudio
hangters
and
aopstudio
authored
add support for NVIDIA llm (infiniflow#1645)
### What problem does this PR solve? add support for NVIDIA llm ### Type of change - [x] New Feature (non-breaking change which adds functionality) --------- Co-authored-by: Zhedong Cen <cenzhedong2@126.com>
1 parent 95821f6 commit b4a281e

8 files changed

Lines changed: 508 additions & 7 deletions

File tree

‎conf/llm_factories.json‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1918,6 +1918,290 @@
19181918
"model_type": "chat"
19191919
}
19201920
]
1921+
},
1922+
{
1923+
"name": "NVIDIA",
1924+
"logo": "",
1925+
"tags": "LLM,TEXT EMBEDDING, TEXT RE-RANK",
1926+
"status": "1",
1927+
"llm": [
1928+
{
1929+
"llm_name": "nvidia/nemotron-4-340b-reward",
1930+
"tags": "LLM,CHAT,4K",
1931+
"max_tokens": 4096,
1932+
"model_type": "chat"
1933+
},
1934+
{
1935+
"llm_name": "aisingapore/sea-lion-7b-instruct",
1936+
"tags": "LLM,CHAT,4K",
1937+
"max_tokens": 4096,
1938+
"model_type": "chat"
1939+
},
1940+
{
1941+
"llm_name": "databricks/dbrx-instruct",
1942+
"tags": "LLM,CHAT,16K",
1943+
"max_tokens": 16384,
1944+
"model_type": "chat"
1945+
},
1946+
{
1947+
"llm_name": "google/gemma-7b",
1948+
"tags": "LLM,CHAT,32K",
1949+
"max_tokens": 32768,
1950+
"model_type": "chat"
1951+
},
1952+
{
1953+
"llm_name": "google/gemma-2b",
1954+
"tags": "LLM,CHAT,16K",
1955+
"max_tokens": 16384,
1956+
"model_type": "chat"
1957+
},
1958+
{
1959+
"llm_name": "google/gemma-2-9b-it",
1960+
"tags": "LLM,CHAT,8K",
1961+
"max_tokens": 8192,
1962+
"model_type": "chat"
1963+
},
1964+
{
1965+
"llm_name": "google/gemma-2-27b-it",
1966+
"tags": "LLM,CHAT,8K",
1967+
"max_tokens": 8192,
1968+
"model_type": "chat"
1969+
},
1970+
{
1971+
"llm_name": "google/recurrentgemma-2b",
1972+
"tags": "LLM,CHAT,4K",
1973+
"max_tokens": 4096,
1974+
"model_type": "chat"
1975+
},
1976+
{
1977+
"llm_name": "mediatek/breeze-7b-instruct",
1978+
"tags": "LLM,CHAT,8K",
1979+
"max_tokens": 8192,
1980+
"model_type": "chat"
1981+
},
1982+
{
1983+
"llm_name": "meta/llama2-70b",
1984+
"tags": "LLM,CHAT,4K",
1985+
"max_tokens": 4096,
1986+
"model_type": "chat"
1987+
},
1988+
{
1989+
"llm_name": "meta/llama3-8b",
1990+
"tags": "LLM,CHAT,8K",
1991+
"max_tokens": 8192,
1992+
"model_type": "chat"
1993+
},
1994+
{
1995+
"llm_name": "meta/llama3-70b",
1996+
"tags": "LLM,CHAT,8K",
1997+
"max_tokens": 8192,
1998+
"model_type": "chat"
1999+
},
2000+
{
2001+
"llm_name": "microsoft/phi-3-medium-128k-instruct",
2002+
"tags": "LLM,CHAT,128K",
2003+
"max_tokens": 131072,
2004+
"model_type": "chat"
2005+
},
2006+
{
2007+
"llm_name": "microsoft/phi-3-medium-4k-instruct",
2008+
"tags": "LLM,CHAT,4K",
2009+
"max_tokens": 4096,
2010+
"model_type": "chat"
2011+
},
2012+
{
2013+
"llm_name": "microsoftphi-3-mini-128k-instruct",
2014+
"tags": "LLM,CHAT,128K",
2015+
"max_tokens": 131072,
2016+
"model_type": "chat"
2017+
},
2018+
{
2019+
"llm_name": "microsoft/phi-3-mini-4k-instruct",
2020+
"tags": "LLM,CHAT,4K",
2021+
"max_tokens": 4096,
2022+
"model_type": "chat"
2023+
},
2024+
{
2025+
"llm_name": "microsoft/phi-3-small-128k-instruct",
2026+
"tags": "LLM,CHAT,128K",
2027+
"max_tokens": 131072,
2028+
"model_type": "chat"
2029+
},
2030+
{
2031+
"llm_name": "microsoft/phi-3-small-8k-instruct",
2032+
"tags": "LLM,CHAT,8K",
2033+
"max_tokens": 8192,
2034+
"model_type": "chat"
2035+
},
2036+
{
2037+
"llm_name": "mistralai/mistral-7b-instruct",
2038+
"tags": "LLM,CHAT,4K",
2039+
"max_tokens": 4096,
2040+
"model_type": "chat"
2041+
},
2042+
{
2043+
"llm_name": "mistralai/mistral-7b-instruct-v0.3",
2044+
"tags": "LLM,CHAT,4K",
2045+
"max_tokens": 4096,
2046+
"model_type": "chat"
2047+
},
2048+
{
2049+
"llm_name": "mistralai/mixtral-8x7b-instruct",
2050+
"tags": "LLM,CHAT,32K",
2051+
"max_tokens": 32768,
2052+
"model_type": "chat"
2053+
},
2054+
{
2055+
"llm_name": "mistralai/mixtral-8x22b-instruct",
2056+
"tags": "LLM,CHAT,64K",
2057+
"max_tokens": 65536,
2058+
"model_type": "chat"
2059+
},
2060+
{
2061+
"llm_name": "mistralai/mistral-large",
2062+
"tags": "LLM,CHAT,32K",
2063+
"max_tokens": 32768,
2064+
"model_type": "chat"
2065+
},
2066+
{
2067+
"llm_name": "nv-mistralai/mistral-nemo-12b-instruct",
2068+
"tags": "LLM,CHAT,128K",
2069+
"max_tokens": 131072,
2070+
"model_type": "chat"
2071+
},
2072+
{
2073+
"llm_name": "nvidia/llama3-chatqa-1.5-70b",
2074+
"tags": "LLM,CHAT,4K",
2075+
"max_tokens": 4096,
2076+
"model_type": "chat"
2077+
},
2078+
{
2079+
"llm_name": "nvidia/llama3-chatqa-1.5-8b",
2080+
"tags": "LLM,CHAT,4K",
2081+
"max_tokens": 4096,
2082+
"model_type": "chat"
2083+
},
2084+
{
2085+
"llm_name": "nvidia/nemotron-4-340b-instruct",
2086+
"tags": "LLM,CHAT,4K",
2087+
"max_tokens": 4096,
2088+
"model_type": "chat"
2089+
},
2090+
{
2091+
"llm_name": "seallms/seallm-7b-v2.5",
2092+
"tags": "LLM,CHAT,4K",
2093+
"max_tokens": 4096,
2094+
"model_type": "chat"
2095+
},
2096+
{
2097+
"llm_name": "snowflake/arctic",
2098+
"tags": "LLM,CHAT,4K",
2099+
"max_tokens": 4096,
2100+
"model_type": "chat"
2101+
},
2102+
{
2103+
"llm_name": "upstage/solar-10.7b-instruct",
2104+
"tags": "LLM,CHAT,4K",
2105+
"max_tokens": 4096,
2106+
"model_type": "chat"
2107+
},
2108+
{
2109+
"llm_name": "baai/bge-m3",
2110+
"tags": "TEXT EMBEDDING,8K",
2111+
"max_tokens": 8192,
2112+
"model_type": "embedding"
2113+
},
2114+
{
2115+
"llm_name": "nvidia/embed-qa-4",
2116+
"tags": "TEXT EMBEDDING,512",
2117+
"max_tokens": 512,
2118+
"model_type": "embedding"
2119+
},
2120+
{
2121+
"llm_name": "nvidia/nv-embed-v1",
2122+
"tags": "TEXT EMBEDDING,32K",
2123+
"max_tokens": 32768,
2124+
"model_type": "embedding"
2125+
},
2126+
{
2127+
"llm_name": "nvidia/nv-embedqa-e5-v5",
2128+
"tags": "TEXT EMBEDDING,512",
2129+
"max_tokens": 512,
2130+
"model_type": "embedding"
2131+
},
2132+
{
2133+
"llm_name": "nvidia/nv-embedqa-mistral-7b-v2",
2134+
"tags": "TEXT EMBEDDING,512",
2135+
"max_tokens": 512,
2136+
"model_type": "embedding"
2137+
},
2138+
{
2139+
"llm_name": "nvidia/nv-rerankqa-mistral-4b-v3",
2140+
"tags": "RE-RANK,512",
2141+
"max_tokens": 512,
2142+
"model_type": "rerank"
2143+
},
2144+
{
2145+
"llm_name": "nvidia/rerank-qa-mistral-4b",
2146+
"tags": "RE-RANK,512",
2147+
"max_tokens": 512,
2148+
"model_type": "rerank"
2149+
},
2150+
{
2151+
"llm_name": "snowflake/arctic-embed-l",
2152+
"tags": "TEXT EMBEDDING,512",
2153+
"max_tokens": 512,
2154+
"model_type": "embedding"
2155+
},
2156+
{
2157+
"llm_name": "adept/fuyu-8b",
2158+
"tags": "LLM,IMAGE2TEXT,4K",
2159+
"max_tokens": 4096,
2160+
"model_type": "image2text"
2161+
},
2162+
{
2163+
"llm_name": "google/deplot",
2164+
"tags": "LLM,IMAGE2TEXT,4K",
2165+
"max_tokens": 4096,
2166+
"model_type": "image2text"
2167+
},
2168+
{
2169+
"llm_name": "google/paligemma",
2170+
"tags": "LLM,IMAGE2TEXT,4K",
2171+
"max_tokens": 4096,
2172+
"model_type": "image2text"
2173+
},
2174+
{
2175+
"llm_name": "Iiuhaotian/Ilava-v1.6-34b",
2176+
"tags": "LLM,IMAGE2TEXT,4K",
2177+
"max_tokens": 4096,
2178+
"model_type": "image2text"
2179+
},
2180+
{
2181+
"llm_name": "Iiuhaotian/Ilava-v1.6-mistral-7b",
2182+
"tags": "LLM,IMAGE2TEXT,4K",
2183+
"max_tokens": 4096,
2184+
"model_type": "image2text"
2185+
},
2186+
{
2187+
"llm_name": "microsoft/kosmos-2",
2188+
"tags": "LLM,IMAGE2TEXT,4K",
2189+
"max_tokens": 4096,
2190+
"model_type": "image2text"
2191+
},
2192+
{
2193+
"llm_name": "microsoft/phi-3-vision-128k-instruct",
2194+
"tags": "LLM,IMAGE2TEXT,128K",
2195+
"max_tokens": 131072,
2196+
"model_type": "image2text"
2197+
},
2198+
{
2199+
"llm_name": "nvidia/neva-22b",
2200+
"tags": "LLM,IMAGE2TEXT,4K",
2201+
"max_tokens": 4096,
2202+
"model_type": "image2text"
2203+
}
2204+
]
19212205
}
19222206
]
19232207
}

‎rag/llm/__init__.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@
3434
"BAAI": DefaultEmbedding,
3535
"Mistral": MistralEmbed,
3636
"Bedrock": BedrockEmbed,
37-
"Gemini":GeminiEmbed
37+
"Gemini":GeminiEmbed,
38+
"NVIDIA":NvidiaEmbed
3839
}
3940

4041

@@ -48,7 +49,8 @@
4849
"Moonshot": LocalCV,
4950
'Gemini':GeminiCV,
5051
'OpenRouter':OpenRouterCV,
51-
"LocalAI":LocalAICV
52+
"LocalAI":LocalAICV,
53+
"NVIDIA":NvidiaCV
5254
}
5355

5456

@@ -71,15 +73,17 @@
7173
"Bedrock": BedrockChat,
7274
"Groq": GroqChat,
7375
'OpenRouter':OpenRouterChat,
74-
"StepFun":StepFunChat
76+
"StepFun":StepFunChat,
77+
"NVIDIA":NvidiaChat
7578
}
7679

7780

7881
RerankModel = {
7982
"BAAI": DefaultRerank,
8083
"Jina": JinaRerank,
8184
"Youdao": YoudaoRerank,
82-
"Xinference": XInferenceRerank
85+
"Xinference": XInferenceRerank,
86+
"NVIDIA":NvidiaRerank
8387
}
8488

8589

0 commit comments

Comments
 (0)