update

OpenSPG · Feb 13, 2025 · b66e0b1 · b66e0b1
1 parent 5300d7a
commit b66e0b1
Show file tree

Hide file tree

Showing 19 changed files with 500 additions and 29 deletions.
diff --git a/kag/examples/FinState/builder/graph_db_tools.py b/kag/examples/FinState/builder/graph_db_tools.py
@@ -1,21 +1,41 @@
+import logging
 from neo4j import GraphDatabase
 
-# 定义数据库连接信息
-uri = "neo4j://localhost:7687"
-username = "neo4j"
-password = "neo4j@openspg"
-# 创建数据库驱动
-driver = GraphDatabase.driver(uri, auth=(username, password))
-
 
 def clear_neo4j_data(db_name):
     """
     清空neo4j数据
     """
 
+    # 定义数据库连接信息
+    uri = "neo4j://localhost:7687"
+    username = "neo4j"
+    password = "neo4j@openspg"
+    # 创建数据库驱动
+    driver = GraphDatabase.driver(uri, auth=(username, password))
+
     def delete_all_nodes_and_relationships(tx):
         # 删除所有节点
         tx.run("MATCH (n) DETACH DELETE n")
 
-    with driver.session(database=db_name) as session:
-        session.execute_write(delete_all_nodes_and_relationships)
+    try:
+        with driver.session(database=db_name) as session:
+            session.execute_write(delete_all_nodes_and_relationships)
+    except:
+        logging.exception("clear db error")
+
+
+def clear_neo4j_data_api():
+    from kag.common.conf import KAG_PROJECT_CONF, KAG_CONFIG
+    import requests
+
+    try:
+        url = KAG_PROJECT_CONF.host_addr + "/graph/clear_db"
+        r_data = {"projectId": KAG_PROJECT_CONF.project_id}
+        requests.post(
+            url,
+            data=r_data,
+            timeout=100,
+        )
+    except:
+        logging.exception("call url error, url=" + url + ",data=" + str(r_data))
diff --git a/kag/examples/TableEval/TableEval/builder/__init__.py b/kag/examples/TableEval/TableEval/builder/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2023 OpenSPG Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied.
+
+"""
+Builder Dir.
+"""
diff --git a/kag/examples/TableEval/TableEval/builder/data/__init__.py b/kag/examples/TableEval/TableEval/builder/data/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2023 OpenSPG Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied.
+
+"""
+Place the files to be used for building the index in this directory.
+"""
diff --git a/kag/examples/TableEval/TableEval/builder/indexer.py b/kag/examples/TableEval/TableEval/builder/indexer.py
@@ -0,0 +1,10 @@
+# Copyright 2023 OpenSPG Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied.
diff --git a/kag/examples/TableEval/TableEval/builder/prompt/__init__.py b/kag/examples/TableEval/TableEval/builder/prompt/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2023 OpenSPG Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied.
+
+"""
+Place the prompts to be used for building the index in this directory.
+"""
diff --git a/kag/examples/TableEval/TableEval/kag_config.yaml b/kag/examples/TableEval/TableEval/kag_config.yaml
@@ -0,0 +1,167 @@
+#------------project configuration start----------------#
+openie_llm: &openie_llm
+  api_key: sk-yndixxjfxvnsqfkvfuyubkxidhtwicjcflprvqguffrmxbrv
+  base_url: https://api.siliconflow.cn/v1/
+  model: Qwen/Qwen2.5-72B-Instruct
+  type: maas
+
+chat_llm: &chat_llm
+  api_key: sk-yndixxjfxvnsqfkvfuyubkxidhtwicjcflprvqguffrmxbrv
+  base_url: https://api.siliconflow.cn/v1/
+  model: Qwen/Qwen2.5-72B-Instruct
+  type: maas
+
+vectorize_model: &vectorize_model
+  api_key: sk-yndixxjfxvnsqfkvfuyubkxidhtwicjcflprvqguffrmxbrv
+  base_url: https://api.siliconflow.cn/v1/
+  model: BAAI/bge-m3
+  type: openai
+  vector_dimensions: 1024
+vectorizer: *vectorize_model
+
+log:
+  level: INFO
+
+project:
+  biz_scene: table
+  #host_addr: http://127.0.0.1:8887
+  host_addr: http://antspg-gz00b-006000164057.sa128-sqa.alipay.net:8887
+  id: '3500002'
+  language: en
+  namespace: TableEval
+#------------project configuration end----------------#
+
+#------------kag-builder configuration start----------------#
+finqa_builder_pipeline:
+  chain:
+    type: unstructured_builder_chain # kag.builder.default_chain.DefaultUnstructuredBuilderChain
+    extractor:
+      type: table_and_text_extractor # kag.builder.component.extractor.schema_free_extractor.SchemaFreeExtractor
+      llm: *openie_llm
+      ner_prompt:
+        type: default_ner # kag.builder.prompt.default.ner.OpenIENERPrompt
+      std_prompt:
+        type: default_std # kag.builder.prompt.default.std.OpenIEEntitystandardizationdPrompt
+      triple_prompt:
+        type: default_triple # kag.builder.prompt.default.triple.OpenIETriplePrompt
+      table_classify_prompt:
+        type: table_classify
+      table_context_prompt:
+        type: table_context
+      table_keywords_prompt:
+        type: table_keywords
+      table_reformat_prompt:
+        type: table_reformat
+    reader:
+      type: md_reader # kag.builder.component.reader.dict_reader.DictReader
+    post_processor:
+      type: kag_post_processor # kag.builder.component.postprocessor.kag_postprocessor.KAGPostProcessor
+      similarity_threshold: 0.9
+    splitter:
+      type: length_splitter # kag.builder.component.splitter.length_splitter.LengthSplitter
+      split_length: 1000
+      window_length: 0
+    vectorizer:
+      type: batch_vectorizer # kag.builder.component.vectorizer.batch_vectorizer.BatchVectorizer
+      vectorize_model: *vectorize_model
+    writer:
+      type: kg_writer # kag.builder.component.writer.kg_writer.KGWriter
+  num_threads_per_chain: 1
+  num_chains: 1
+  scanner:
+    type: file_scanner # kag.builder.component.scanner.dataset_scanner.HotpotqaCorpusScanner
+
+kag_chunk_builder_pipeline:
+  chain:
+    type: unstructured_builder_chain # kag.builder.default_chain.DefaultUnstructuredBuilderChain
+    extractor:
+      type: schema_free_extractor # kag.builder.component.extractor.schema_free_extractor.SchemaFreeExtractor
+      llm: *openie_llm
+      ner_prompt:
+        type: default_ner # kag.builder.prompt.default.ner.OpenIENERPrompt
+      std_prompt:
+        type: default_std # kag.builder.prompt.default.std.OpenIEEntitystandardizationdPrompt
+      triple_prompt:
+        type: default_triple # kag.builder.prompt.default.triple.OpenIETriplePrompt
+    reader:
+      type: md_reader # kag.builder.component.reader.dict_reader.DictReader
+    post_processor:
+      type: kag_post_processor # kag.builder.component.postprocessor.kag_postprocessor.KAGPostProcessor
+      similarity_threshold: 0.9
+    splitter:
+      type: length_splitter # kag.builder.component.splitter.length_splitter.LengthSplitter
+      split_length: 100000
+      window_length: 0
+    vectorizer:
+      type: batch_vectorizer # kag.builder.component.vectorizer.batch_vectorizer.BatchVectorizer
+      vectorize_model: *vectorize_model
+    writer:
+      type: kg_writer # kag.builder.component.writer.kg_writer.KGWriter
+  num_threads_per_chain: 1
+  num_chains: 1
+  scanner:
+    type: file_scanner # kag.builder.component.scanner.dataset_scanner.MusiqueCorpusScanner
+#------------kag-builder configuration end----------------#
+
+#------------kag-solver configuration start----------------#
+search_api: &search_api
+  type: openspg_search_api #kag.solver.tools.search_api.impl.openspg_search_api.OpenSPGSearchAPI
+
+graph_api: &graph_api
+  type: openspg_graph_api #kag.solver.tools.graph_api.impl.openspg_graph_api.OpenSPGGraphApi
+
+exact_kg_retriever: &exact_kg_retriever
+  type: default_exact_kg_retriever # kag.solver.retriever.impl.default_exact_kg_retriever.DefaultExactKgRetriever
+  el_num: 5
+  llm_client: *chat_llm
+  search_api: *search_api
+  graph_api: *graph_api
+
+fuzzy_kg_retriever: &fuzzy_kg_retriever
+  type: default_fuzzy_kg_retriever # kag.solver.retriever.impl.default_fuzzy_kg_retriever.DefaultFuzzyKgRetriever
+  el_num: 5
+  vectorize_model: *vectorize_model
+  llm_client: *chat_llm
+  search_api: *search_api
+  graph_api: *graph_api
+
+chunk_retriever: &chunk_retriever
+  type: default_chunk_retriever # kag.solver.retriever.impl.default_fuzzy_kg_retriever.DefaultFuzzyKgRetriever
+  llm_client: *chat_llm
+  ner_prompt:
+    type: table_question_ner
+  recall_num: 10
+  rerank_topk: 10
+
+kag_solver_pipeline:
+  memory:
+    type: default_memory # kag.solver.implementation.default_memory.DefaultMemory
+    llm_client: *chat_llm
+  max_iterations: 3
+  reasoner:
+    type: default_reasoner # kag.solver.implementation.default_reasoner.DefaultReasoner
+    llm_client: *chat_llm
+    lf_planner:
+      type: default_lf_planner # kag.solver.plan.default_lf_planner.DefaultLFPlanner
+      llm_client: *chat_llm
+      vectorize_model: *vectorize_model
+      logic_form_plan_prompt:
+        type: only_retriever_plan
+    lf_executor:
+      type: default_lf_executor # kag.solver.execute.default_lf_executor.DefaultLFExecutor
+      llm_client: *chat_llm
+      force_chunk_retriever: true
+      exact_kg_retriever: *exact_kg_retriever
+      fuzzy_kg_retriever: *fuzzy_kg_retriever
+      chunk_retriever: *chunk_retriever
+      merger:
+        type: default_lf_sub_query_res_merger # kag.solver.execute.default_sub_query_merger.DefaultLFSubQueryResMerger
+        vectorize_model: *vectorize_model
+        chunk_retriever: *chunk_retriever
+  generator:
+    type: default_generator # kag.solver.implementation.default_generator.DefaultGenerator
+    llm_client: *chat_llm
+  reflector:
+    type: default_reflector # kag.solver.implementation.default_reflector.DefaultReflector
+    llm_client: *chat_llm
+#------------kag-solver configuration end----------------#
diff --git a/kag/examples/TableEval/TableEval/reasoner/__init__.py b/kag/examples/TableEval/TableEval/reasoner/__init__.py
@@ -0,0 +1,20 @@
+# Copyright 2023 OpenSPG Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied.
+
+"""
+Place the DSL file for graph reasoning in this directory.
+For example:
+
+```company.dsl
+MATCH (s:DEFAULT.Company)
+RETURN s.id, s.address
+```
+"""