diff --git a/config/main.sample.toml b/config/main.sample.toml index fb966d87..7c730f63 100644 --- a/config/main.sample.toml +++ b/config/main.sample.toml @@ -14,14 +14,14 @@ url = "https://proof-service.next.id" api_key = "x-api-key" [upstream.aggregation_service] -url = "https://7x16bogxfb.execute-api.us-east-1.amazonaws.com/v1/identity/search" +url = "http://data-server-hostname/data_server" [upstream.sybil_service] url = "https://raw.githubusercontent.com/Uniswap/sybil-list/master/verified.json" [upstream.keybase_service] url = "https://keybase.io/_/api/1.0/user/lookup.json" -stable_url = "http://ec2-18-167-90-166.ap-east-1.compute.amazonaws.com/data_server/keybase" +stable_url = "http://data-server-hostname/data_server/keybase" [upstream.knn3_service] url = "https://mw.graphql.knn3.xyz/" @@ -62,4 +62,4 @@ url = "https://indexer.crossbell.io/v1/graphql" rpc_url = "https://api.mainnet-beta.solana.com" [upstream.genome_api] -rpc_url = "http://ec2-18-167-90-166.ap-east-1.compute.amazonaws.com/data_server/genome" +rpc_url = "http://data-server-hostname/data_server/genome" diff --git a/src/config/tdb/migrations/LoadingJob_SocialGraph.gsql b/src/config/tdb/migrations/LoadingJob_SocialGraph.gsql index 5e17b7be..497b8822 100644 --- a/src/config/tdb/migrations/LoadingJob_SocialGraph.gsql +++ b/src/config/tdb/migrations/LoadingJob_SocialGraph.gsql @@ -2,6 +2,61 @@ CREATE GRAPH SocialGraph (Identities, Proof_Forward, Proof_Backward, Contracts, USE GRAPH SocialGraph +CREATE OR REPLACE QUERY insert_contract_connection(STRING edges_str) FOR GRAPH SocialGraph SYNTAX v2 { + JSONARRAY edges = parse_json_array(edges_str); + SumAccum @@created_edges; + INT array_size = edges.size(); + FOREACH idx IN RANGE[0, array_size - 1] DO + JSONOBJECT edge_obj = edges.getJsonObject(idx); + STRING edge_type = edge_obj.getString("edge_type"); + STRING from_id = edge_obj.getString("from_id"); + STRING to_id = edge_obj.getString("to_id"); + IF edge_type == "Hold_Contract" THEN + INSERT INTO Hold_Contract(FROM, TO, DISCRIMINATOR(source, transaction, id), uuid, created_at, updated_at, fetcher, expired_at) + VALUES ( + from_id Identities, + to_id Contracts, + edge_obj.getString("source"), + edge_obj.getString("transaction"), + edge_obj.getString("id"), + edge_obj.getString("uuid"), + to_datetime(edge_obj.getString("created_at")), + now(), + edge_obj.getString("fetcher"), + to_datetime(edge_obj.getString("expired_at")) + ); + @@created_edges += 1; + ELSE IF edge_type == "Reverse_Resolve_Contract" THEN + INSERT INTO Reverse_Resolve_Contract(FROM, TO, DISCRIMINATOR(source, system, name), uuid, updated_at, fetcher) + VALUES ( + from_id Identities, + to_id Contracts, + edge_obj.getString("source"), + edge_obj.getString("system"), + edge_obj.getString("name"), + edge_obj.getString("uuid"), + now(), + edge_obj.getString("fetcher") + ); + @@created_edges += 1; + ELSE IF edge_type == "Resolve_Contract" THEN + INSERT INTO Resolve_Contract(FROM, TO, DISCRIMINATOR(source, system, name), uuid, updated_at, fetcher) + VALUES ( + from_id Contracts, + to_id Identities, + edge_obj.getString("source"), + edge_obj.getString("system"), + edge_obj.getString("name"), + edge_obj.getString("uuid"), + now(), + edge_obj.getString("fetcher") + ); + @@created_edges += 1; + END; + END; + PRINT @@created_edges as created_edges; +} + CREATE OR REPLACE QUERY upsert_isolated_vertex(STRING vertex_str, INT updated_nanosecond) FOR GRAPH SocialGraph SYNTAX v2 { TYPEDEF TUPLE< INT updated_nanosecond, STRING id > MinUpdatedTimeTuple; JSONOBJECT from_v = parse_json_object(vertex_str); diff --git a/src/tigergraph/edge/hold.rs b/src/tigergraph/edge/hold.rs index 91d5a696..14b9fd29 100644 --- a/src/tigergraph/edge/hold.rs +++ b/src/tigergraph/edge/hold.rs @@ -225,7 +225,7 @@ impl Transfer for HoldRecord { attributes_map } - fn to_json_value(&self) -> Value { + fn to_json_value(&self) -> Map { let mut map = Map::new(); map.insert("uuid".to_string(), json!(self.uuid)); map.insert("source".to_string(), json!(self.source)); @@ -246,7 +246,7 @@ impl Transfer for HoldRecord { self.expired_at .map_or(json!("1970-01-01 00:00:00"), |expired_at| json!(expired_at)), ); - Value::Object(map) + map } } diff --git a/src/tigergraph/edge/mod.rs b/src/tigergraph/edge/mod.rs index bf8ac771..b5dd36ec 100644 --- a/src/tigergraph/edge/mod.rs +++ b/src/tigergraph/edge/mod.rs @@ -1,8 +1,10 @@ pub mod hold; +pub mod part_of_identities_graph; pub mod proof; pub mod relation; pub mod resolve; pub use hold::{Hold, HoldRecord, HOLD_CONTRACT, HOLD_IDENTITY}; +pub use part_of_identities_graph::{HyperEdge, HyperEdgeRecord, HYPER_EDGE, HYPER_EDGE_REVERSE}; pub use proof::{ Proof, ProofRecord, EDGE_NAME as PROOF_EDGE, REVERSE_EDGE_NAME as PROOF_REVERSE_EDGE, }; diff --git a/src/tigergraph/edge/part_of_identities_graph.rs b/src/tigergraph/edge/part_of_identities_graph.rs new file mode 100644 index 00000000..09f63319 --- /dev/null +++ b/src/tigergraph/edge/part_of_identities_graph.rs @@ -0,0 +1,174 @@ +use crate::{ + error::Error, + tigergraph::{ + edge::{Edge, EdgeRecord, EdgeWrapper, FromWithParams, Wrapper}, + vertex::{IdentitiesGraph, Identity, Vertex, VertexRecord}, + Attribute, Transfer, + }, +}; + +use hyper::{client::HttpConnector, Client}; +use serde::{Deserialize, Serialize}; +use serde_json::value::{Map, Value}; +use std::collections::HashMap; +use uuid::Uuid; + +// always IdentitiesGraph -> Identities +pub const HYPER_EDGE: &str = "PartOfIdentitiesGraph_Reverse"; +pub const HYPER_EDGE_REVERSE: &str = "PartOfIdentitiesGraph_Reverse"; +pub const IS_DIRECTED: bool = true; + +/// HyperEdge +#[derive(Clone, Deserialize, Serialize, Debug)] +pub struct HyperEdge {} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct HyperEdgeRecord(pub EdgeRecord); + +impl FromWithParams for EdgeRecord { + fn from_with_params( + e_type: String, + directed: bool, + from_id: String, + from_type: String, + to_id: String, + to_type: String, + attributes: HyperEdge, + ) -> Self { + EdgeRecord { + e_type, + directed, + from_id, + from_type, + to_id, + to_type, + discriminator: None, + attributes, + } + } +} + +impl From> for HyperEdgeRecord { + fn from(record: EdgeRecord) -> Self { + HyperEdgeRecord(record) + } +} + +impl std::ops::Deref for HyperEdgeRecord { + type Target = EdgeRecord; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for HyperEdgeRecord { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl std::ops::Deref for EdgeRecord { + type Target = HyperEdge; + + fn deref(&self) -> &Self::Target { + &self.attributes + } +} + +impl std::ops::DerefMut for EdgeRecord { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.attributes + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct HyperEdgeAttribute(HashMap); + +// Implement the `From` trait for converting `HyperEdgeRecord` into a `HashMap`. +impl Transfer for HyperEdgeRecord { + fn to_attributes_map(&self) -> HashMap { + let attributes_map = HashMap::new(); + attributes_map + } + + fn to_json_value(&self) -> Map { + let map = Map::new(); + map + } +} + +#[async_trait::async_trait] +impl Edge for HyperEdgeRecord { + fn e_type(&self) -> String { + self.e_type.clone() + } + + fn directed(&self) -> bool { + // TODO: query from server is the best solution + self.directed.clone() + } + + /// Find an edge by UUID. + async fn find_by_uuid( + _client: &Client, + _uuid: &Uuid, + ) -> Result, Error> { + todo!() + } + + /// Find `EdgeRecord` by source and target + async fn find_by_from_to( + &self, + _client: &Client, + _from: &VertexRecord, + _to: &VertexRecord, + _filter: Option>, + ) -> Result>, Error> { + todo!() + } + + /// Connect 2 vertex. + async fn connect( + &self, + _client: &Client, + _from: &IdentitiesGraph, + _to: &Identity, + ) -> Result<(), Error> { + todo!() + } + + /// notice this function is deprecated + async fn connect_reverse( + &self, + _client: &Client, + _from: &IdentitiesGraph, + _to: &Identity, + ) -> Result<(), Error> { + todo!() + } +} + +impl Wrapper for HyperEdge { + fn wrapper( + &self, + from: &IdentitiesGraph, + to: &Identity, + name: &str, + ) -> EdgeWrapper { + let part_of = EdgeRecord::from_with_params( + name.to_string(), + IS_DIRECTED, + from.primary_key(), + from.vertex_type(), + to.primary_key(), + to.vertex_type(), + self.to_owned(), + ); + EdgeWrapper { + edge: HyperEdgeRecord(part_of), + source: from.to_owned(), + target: to.to_owned(), + } + } +} diff --git a/src/tigergraph/edge/proof.rs b/src/tigergraph/edge/proof.rs index ef9923a4..bdac6220 100644 --- a/src/tigergraph/edge/proof.rs +++ b/src/tigergraph/edge/proof.rs @@ -191,7 +191,7 @@ impl Transfer for ProofRecord { attributes_map } - fn to_json_value(&self) -> Value { + fn to_json_value(&self) -> Map { let mut map = Map::new(); map.insert("uuid".to_string(), json!(self.uuid)); map.insert("source".to_string(), json!(self.source)); @@ -207,7 +207,7 @@ impl Transfer for ProofRecord { ); map.insert("updated_at".to_string(), json!(self.updated_at)); map.insert("fetcher".to_string(), json!(self.fetcher)); - Value::Object(map) + map } } diff --git a/src/tigergraph/edge/resolve.rs b/src/tigergraph/edge/resolve.rs index f51f36da..a37b10ed 100644 --- a/src/tigergraph/edge/resolve.rs +++ b/src/tigergraph/edge/resolve.rs @@ -187,7 +187,7 @@ impl Transfer for ResolveRecord { ); attributes_map } - fn to_json_value(&self) -> Value { + fn to_json_value(&self) -> Map { let mut map = Map::new(); map.insert("uuid".to_string(), json!(self.uuid)); map.insert("source".to_string(), json!(self.source)); @@ -195,7 +195,7 @@ impl Transfer for ResolveRecord { map.insert("name".to_string(), json!(self.name)); map.insert("fetcher".to_string(), json!(self.fetcher)); map.insert("updated_at".to_string(), json!(self.updated_at)); - Value::Object(map) + map } } diff --git a/src/tigergraph/mod.rs b/src/tigergraph/mod.rs index c6b8a305..05005bff 100644 --- a/src/tigergraph/mod.rs +++ b/src/tigergraph/mod.rs @@ -7,24 +7,30 @@ use crate::{ config::C, error::Error, tigergraph::{ - edge::{Edge, Hold, Proof, Resolve, Wrapper}, edge::{ - HOLD_CONTRACT, HOLD_IDENTITY, PROOF_EDGE, PROOF_REVERSE_EDGE, RESOLVE, - RESOLVE_CONTRACT, REVERSE_RESOLVE, REVERSE_RESOLVE_CONTRACT, + Edge, Hold, HoldRecord, HyperEdgeRecord, Proof, ProofRecord, Resolve, ResolveRecord, + Wrapper, }, - vertex::{Contract, Identity, Vertex}, + edge::{ + HOLD_CONTRACT, HOLD_IDENTITY, HYPER_EDGE_REVERSE, PROOF_EDGE, PROOF_REVERSE_EDGE, + RESOLVE, RESOLVE_CONTRACT, REVERSE_RESOLVE, REVERSE_RESOLVE_CONTRACT, + }, + vertex::{Contract, IdentitiesGraph, Identity, Vertex}, }, - util::parse_body, + util::{make_client, parse_body}, }; use http::uri::InvalidUri; use hyper::Method; use hyper::{client::HttpConnector, Body, Client}; use serde::{Deserialize, Serialize}; -use serde_json::value::Value; +use serde_json::json; +use serde_json::value::{Map, Value}; use std::collections::HashMap; +use std::convert::{TryFrom, TryInto}; use strum_macros::{Display, EnumIter, EnumString}; use tracing::{error, trace}; +use uuid::Uuid; #[derive( Serialize, @@ -111,6 +117,189 @@ impl Graph { } } +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct IdAllocation { + pub graph_id: String, + pub updated_nanosecond: i64, // microseconds are one-millionth of a second (1/1,000,000 seconds) + pub vids: Vec, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct IdAllocationResponse { + pub code: i32, + pub msg: Option, + pub data: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct IdAllocationResult { + #[serde(rename = "return_graph_id")] + pub graph_id: String, + #[serde(rename = "return_updated_nanosecond")] + pub updated_nanosecond: i64, +} + +pub async fn id_allocation(payload: &IdAllocation) -> Result { + let http_client = make_client(); + let id_allocation_url = format!("{}:{}", C.tdb.host.trim_end_matches(":9000"), "9002"); + let uri: http::Uri = format!("{}/id_allocation/allocation", id_allocation_url,) + .parse() + .map_err(|_err: InvalidUri| Error::ParamError(format!("Uri format Error {}", _err)))?; + + let json_params = serde_json::to_string(payload).map_err(|err| Error::JSONParseError(err))?; + let req = hyper::Request::builder() + .method(Method::POST) + .uri(uri) + .body(Body::from(json_params)) + .map_err(|_err| Error::ParamError(format!("ParamError Error {}", _err)))?; + + let mut resp = http_client.request(req).await.map_err(|err| { + Error::ManualHttpClientError(format!( + "TigerGraph | Fail call allocation: {:?}", + err.to_string() + )) + })?; + + if !resp.status().is_success() { + let err_message = format!( + "TigerGraph | Fail call allocation, statusCode: {}", + resp.status() + ); + error!(err_message); + return Err(Error::General(err_message, resp.status())); + } + + let data = match parse_body::(&mut resp).await { + Ok(result) => { + if result.code != 0 { + let err_resp: UpsertGraphResponse = parse_body(&mut resp).await?; + let err_message = format!( + "TigerGraph | Fail call allocation: Code: {:?}, Message: {:?}", + err_resp.base.code, err_resp.base.message + ); + error!(err_message); + return Err(Error::General(err_message, resp.status())); + } + result.data + } + Err(err) => { + let err_message = format!("TigerGraph | call allocation parse_body error: {:?}", err); + error!(err_message); + return Err(Error::General(err_message, resp.status())); + } + }; + + match data { + Some(data) => { + if data.graph_id == "".to_string() || data.updated_nanosecond == 0 { + return Err(Error::ParamMissing( + "allocation graph_id | updated_nanosecond missing".to_string(), + )); + } + Ok(data) + } + None => Err(Error::ParamMissing("allocation body missing".to_string())), + } +} + +pub async fn batch_upsert( + client: &Client, + edges: Vec, +) -> Result<(), Error> { + // let json_raw = serde_json::to_string(&edges).map_err(|err| Error::JSONParseError(err))?; + // trace!("edges = {}", json_raw); + let mut graph: UpsertGraph = BatchEdges(edges.clone()).into(); + // let json_raw_2 = serde_json::to_string(&graph).map_err(|err| Error::JSONParseError(err))?; + // trace!("Graph upsert struct: {}", json_raw_2); + let vids = graph.extract_connected_vertices_ids(); + trace!("Connected Identities vids: {:?}", vids); + let generate_id = Uuid::new_v4().to_string(); + let updated_nanosecond = chrono::Utc::now().naive_utc().and_utc().timestamp_micros(); + let allocation_req = IdAllocation { + graph_id: generate_id.clone(), + updated_nanosecond: updated_nanosecond.clone(), + vids, + }; + + let mut final_identity_graph = generate_id.clone(); + let mut final_updated_nanosecond: i64 = updated_nanosecond.clone(); + match id_allocation(&allocation_req).await { + Ok(result) => { + if generate_id != result.graph_id { + trace!( + "Use Allocation ID: allocation_id({}, nano={})", + result.graph_id, + result.updated_nanosecond + ); + final_identity_graph = result.graph_id; + final_updated_nanosecond = result.updated_nanosecond; + } + } + Err(err) => { + trace!( + "Error during Allocation ID: {:?}, using generate_id({}, nano={})", + err, + generate_id, + updated_nanosecond, + ); + final_identity_graph = generate_id; + final_updated_nanosecond = updated_nanosecond; + } + } + // trace!("final_identity_graph: {:?}", final_identity_graph); + graph.replace_fake_graph_id(&final_identity_graph, final_updated_nanosecond); + // let json_raw = serde_json::to_string(&graph).map_err(|err| Error::JSONParseError(err))?; + // trace!("graph = {}", json_raw); + upsert_graph(client, &graph, Graph::SocialGraph).await?; + let contracts_req: ContractEdgesRequest = BatchEdges(edges).try_into()?; + insert_contract_connection(client, &contracts_req, Graph::SocialGraph).await?; + Ok(()) +} + +// ContractConnectionsResponse +pub async fn insert_contract_connection( + client: &Client, + payload: &ContractEdgesRequest, + graph_name: Graph, +) -> Result<(), Error> { + let uri: http::Uri = format!( + "{}/query/{}/insert_contract_connection", + C.tdb.host, + graph_name.to_string(), + ) + .parse() + .map_err(|_err: InvalidUri| Error::ParamError(format!("Uri format Error {}", _err)))?; + + let json_params = serde_json::to_string(&payload).map_err(|err| Error::JSONParseError(err))?; + let req = hyper::Request::builder() + .method(Method::POST) + .uri(uri) + .header("Authorization", graph_name.token()) + .body(Body::from(json_params)) + .map_err(|_err| Error::ParamError(format!("ParamError Error {}", _err)))?; + let mut resp = client.request(req).await.map_err(|err| { + Error::ManualHttpClientError(format!( + "TigerGraph | Fail to insert_contract_connection: {:?}", + err.to_string() + )) + })?; + let _result = match parse_body::(&mut resp).await { + Ok(result) => result, + Err(_) => { + let err_resp: ContractConnectionsResponse = parse_body(&mut resp).await?; + let err_message = format!( + "TigerGraph fail to insert_contract_connection, Code: {:?}, Message: {:?}", + err_resp.base.code, err_resp.base.message + ); + error!(err_message); + return Err(Error::General(err_message, resp.status())); + } + }; + let json_raw = serde_json::to_string(&_result).map_err(|err| Error::JSONParseError(err))?; + trace!("TigerGraph insert_contract_connection {}...", json_raw); + Ok(()) +} + pub async fn delete_vertex_and_edge( client: &Client, v_id: String, @@ -202,8 +391,8 @@ pub async fn upsert_graph( return Err(Error::General(err_message, resp.status())); } }; - // let json_raw = serde_json::to_string(&result).map_err(|err| Error::JSONParseError(err))?; - // println!("{}", json_raw); + let json_raw = serde_json::to_string(&_result).map_err(|err| Error::JSONParseError(err))?; + trace!("{}", json_raw); trace!("TigerGraph UpsertGraph ..."); Ok(()) } @@ -223,6 +412,55 @@ pub struct UpsertGraph { >, } +impl UpsertGraph { + /// Extract all primary IDs from the `vertices` map. + pub fn extract_all_vertices_ids(&self) -> Vec { + self.vertices + .get("Identities") + .map(|identities_map| identities_map.keys().cloned().collect()) + .unwrap_or_else(Vec::new) + } + + pub fn extract_connected_vertices_ids(&self) -> Vec { + if let Some(edges) = &self.edges { + if let Some(identities_graph_edges) = edges.get("IdentitiesGraph") { + if let Some(edge_map) = identities_graph_edges.get("fake_uuid_v4") { + if let Some(part_of_reverse_map) = edge_map.get("PartOfIdentitiesGraph_Reverse") + { + if let Some(identities_map) = part_of_reverse_map.get("Identities") { + return identities_map.keys().cloned().collect(); + } + } + } + } + } + Vec::new() + } + + pub fn replace_fake_graph_id(&mut self, new_id: &str, updated_nanosecond: i64) { + if let Some(identities_graph) = self.vertices.get_mut("IdentitiesGraph") { + if let Some(mut attributes_map) = identities_graph.remove("fake_uuid_v4") { + if let Some(id_attr) = attributes_map.get_mut("id") { + id_attr.value = json!(new_id.to_string()); + } + if let Some(updated_nanosecond_attr) = attributes_map.get_mut("updated_nanosecond") + { + updated_nanosecond_attr.value = json!(updated_nanosecond); + } + identities_graph.insert(new_id.to_string(), attributes_map); + } + } + + if let Some(edges) = self.edges.as_mut() { + if let Some(identities_graph_edges) = edges.get_mut("IdentitiesGraph") { + if let Some(edges_map) = identities_graph_edges.remove("fake_uuid_v4") { + identities_graph_edges.insert(new_id.to_string(), edges_map); + } + } + } + } +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct BaseResponse { pub error: bool, @@ -237,6 +475,18 @@ struct UpsertGraphResponse { results: Option>, } +#[derive(Debug, Clone, Deserialize, Serialize)] +struct ContractConnectionsResponse { + #[serde(flatten)] + base: BaseResponse, + results: Option>, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +struct ContractConnectionsResult { + created_edges: i32, +} + #[derive(Debug, Clone, Deserialize, Serialize)] struct UpsertResult { accepted_vertices: i32, @@ -278,7 +528,7 @@ where pub trait Transfer { fn to_attributes_map(&self) -> HashMap; - fn to_json_value(&self) -> Value; + fn to_json_value(&self) -> Map; } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -301,35 +551,367 @@ where let mut edges_map = HashMap::new(); let mut vertices_map: HashMap>> = HashMap::new(); + for edge_wrapper in edges.0 { - let target_vertex_id: HashMap> = HashMap::from([( - edge_wrapper.target.primary_key(), - edge_wrapper.edge.to_attributes_map(), - )]); - let target_vertex_type = - HashMap::from([(edge_wrapper.target.vertex_type(), target_vertex_id)]); - let edge_type_map = HashMap::from([(edge_wrapper.edge.e_type(), target_vertex_type)]); - let source_vertex_type = edges_map - .entry(edge_wrapper.source.vertex_type()) - .or_insert(HashMap::new()); - source_vertex_type.insert(edge_wrapper.source.primary_key(), edge_type_map); + let source_vertex_type = edge_wrapper.source.vertex_type(); + let source_vertex_id = edge_wrapper.source.primary_key(); + let edge_type = edge_wrapper.edge.e_type(); + let target_vertex_type = edge_wrapper.target.vertex_type(); + let target_vertex_id = edge_wrapper.target.primary_key(); + let edge_attributes = edge_wrapper.edge.to_attributes_map(); + + edges_map + .entry(source_vertex_type.clone()) + .or_insert_with(HashMap::new) + .entry(source_vertex_id.clone()) + .or_insert_with(HashMap::new) + .entry(edge_type.clone()) + .or_insert_with(HashMap::new) + .entry(target_vertex_type.clone()) + .or_insert_with(HashMap::new) + .insert(target_vertex_id.clone(), edge_attributes); // Insert source data + vertices_map + .entry(source_vertex_type.clone()) + .or_insert_with(HashMap::new) + .insert( + source_vertex_id.clone(), + edge_wrapper.source.to_attributes_map(), + ); + + // Insert target data + vertices_map + .entry(target_vertex_type.clone()) + .or_insert_with(HashMap::new) + .insert( + target_vertex_id.clone(), + edge_wrapper.target.to_attributes_map(), + ); + } + + UpsertGraph { + vertices: vertices_map, + edges: Some(edges_map), + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub enum EdgeWrapperEnum { + ProofForward(EdgeWrapper), + ProofBackward(EdgeWrapper), + HoldIdentity(EdgeWrapper), + HoldContract(EdgeWrapper), + Resolve(EdgeWrapper), + ReverseResolve(EdgeWrapper), + ResolveContract(EdgeWrapper), + ReverseResolveContract(EdgeWrapper), + PartOfIdentitiesGraph(EdgeWrapper), +} + +impl Transfer for EdgeWrapperEnum { + fn to_attributes_map(&self) -> HashMap { + match self { + EdgeWrapperEnum::ProofForward(wrapper) => wrapper.edge.to_attributes_map(), + EdgeWrapperEnum::ProofBackward(wrapper) => wrapper.edge.to_attributes_map(), + EdgeWrapperEnum::HoldIdentity(wrapper) => wrapper.edge.to_attributes_map(), + EdgeWrapperEnum::HoldContract(wrapper) => wrapper.edge.to_attributes_map(), + EdgeWrapperEnum::Resolve(wrapper) => wrapper.edge.to_attributes_map(), + EdgeWrapperEnum::ReverseResolve(wrapper) => wrapper.edge.to_attributes_map(), + EdgeWrapperEnum::ResolveContract(wrapper) => wrapper.edge.to_attributes_map(), + EdgeWrapperEnum::ReverseResolveContract(wrapper) => wrapper.edge.to_attributes_map(), + EdgeWrapperEnum::PartOfIdentitiesGraph(wrapper) => wrapper.edge.to_attributes_map(), + } + } + + fn to_json_value(&self) -> Map { + match self { + EdgeWrapperEnum::ProofForward(wrapper) => wrapper.edge.to_json_value(), + EdgeWrapperEnum::ProofBackward(wrapper) => wrapper.edge.to_json_value(), + EdgeWrapperEnum::HoldIdentity(wrapper) => wrapper.edge.to_json_value(), + EdgeWrapperEnum::HoldContract(wrapper) => wrapper.edge.to_json_value(), + EdgeWrapperEnum::Resolve(wrapper) => wrapper.edge.to_json_value(), + EdgeWrapperEnum::ReverseResolve(wrapper) => wrapper.edge.to_json_value(), + EdgeWrapperEnum::ResolveContract(wrapper) => wrapper.edge.to_json_value(), + EdgeWrapperEnum::ReverseResolveContract(wrapper) => wrapper.edge.to_json_value(), + EdgeWrapperEnum::PartOfIdentitiesGraph(wrapper) => wrapper.edge.to_json_value(), + } + } +} + +impl EdgeWrapperEnum { + pub fn source(&self) -> &dyn Vertex { + match self { + EdgeWrapperEnum::ProofForward(wrapper) => &wrapper.source, + EdgeWrapperEnum::ProofBackward(wrapper) => &wrapper.source, + EdgeWrapperEnum::HoldIdentity(wrapper) => &wrapper.source, + EdgeWrapperEnum::HoldContract(wrapper) => &wrapper.source, + EdgeWrapperEnum::Resolve(wrapper) => &wrapper.source, + EdgeWrapperEnum::ReverseResolve(wrapper) => &wrapper.source, + EdgeWrapperEnum::ResolveContract(wrapper) => &wrapper.source, + EdgeWrapperEnum::ReverseResolveContract(wrapper) => &wrapper.source, + EdgeWrapperEnum::PartOfIdentitiesGraph(wrapper) => &wrapper.source, + } + } + + pub fn target(&self) -> &dyn Vertex { + match self { + EdgeWrapperEnum::ProofForward(wrapper) => &wrapper.target, + EdgeWrapperEnum::ProofBackward(wrapper) => &wrapper.target, + EdgeWrapperEnum::HoldIdentity(wrapper) => &wrapper.target, + EdgeWrapperEnum::HoldContract(wrapper) => &wrapper.target, + EdgeWrapperEnum::Resolve(wrapper) => &wrapper.target, + EdgeWrapperEnum::ReverseResolve(wrapper) => &wrapper.target, + EdgeWrapperEnum::ResolveContract(wrapper) => &wrapper.target, + EdgeWrapperEnum::ReverseResolveContract(wrapper) => &wrapper.target, + EdgeWrapperEnum::PartOfIdentitiesGraph(wrapper) => &wrapper.target, + } + } + + pub fn e_type(&self) -> &str { + match self { + EdgeWrapperEnum::ProofForward(_) => PROOF_EDGE, + EdgeWrapperEnum::ProofBackward(_) => PROOF_REVERSE_EDGE, + EdgeWrapperEnum::HoldIdentity(_) => HOLD_IDENTITY, + EdgeWrapperEnum::HoldContract(_) => HOLD_CONTRACT, + EdgeWrapperEnum::Resolve(_) => RESOLVE, + EdgeWrapperEnum::ReverseResolve(_) => REVERSE_RESOLVE, + EdgeWrapperEnum::ResolveContract(_) => RESOLVE_CONTRACT, + EdgeWrapperEnum::ReverseResolveContract(_) => REVERSE_RESOLVE_CONTRACT, + EdgeWrapperEnum::PartOfIdentitiesGraph(_) => HYPER_EDGE_REVERSE, + } + } +} + +impl EdgeWrapperEnum { + pub fn new_proof_forward(wrapper: EdgeWrapper) -> Self { + EdgeWrapperEnum::ProofForward(wrapper) + } + + pub fn new_proof_backward(wrapper: EdgeWrapper) -> Self { + EdgeWrapperEnum::ProofBackward(wrapper) + } + + pub fn new_hold_identity(wrapper: EdgeWrapper) -> Self { + EdgeWrapperEnum::HoldIdentity(wrapper) + } + + pub fn new_hold_contract(wrapper: EdgeWrapper) -> Self { + EdgeWrapperEnum::HoldContract(wrapper) + } + + pub fn new_resolve(wrapper: EdgeWrapper) -> Self { + EdgeWrapperEnum::Resolve(wrapper) + } + + pub fn new_reverse_resolve(wrapper: EdgeWrapper) -> Self { + EdgeWrapperEnum::ReverseResolve(wrapper) + } + + pub fn new_resolve_contract(wrapper: EdgeWrapper) -> Self { + EdgeWrapperEnum::ResolveContract(wrapper) + } + + pub fn new_reverse_resolve_contract( + wrapper: EdgeWrapper, + ) -> Self { + EdgeWrapperEnum::ReverseResolveContract(wrapper) + } + + pub fn new_hyper_edge( + wrapper: EdgeWrapper, + ) -> Self { + EdgeWrapperEnum::PartOfIdentitiesGraph(wrapper) + } +} + +/// List edges. +pub type EdgeList = Vec; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct BatchEdges(pub EdgeList); + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ContractEdgesRequest { + pub edges_str: String, +} + +impl TryFrom for ContractEdgesRequest { + type Error = Error; + fn try_from(edges: BatchEdges) -> Result { + let mut connections: Vec = Vec::new(); + for edge_wrapper_enum in edges.0 { + let edge_type = edge_wrapper_enum.e_type(); + if edge_type == HOLD_CONTRACT + || edge_type == RESOLVE_CONTRACT + || edge_type == REVERSE_RESOLVE_CONTRACT { - let outer_map_key = edge_wrapper.source.vertex_type().clone(); - let inner_map_key = edge_wrapper.source.primary_key().clone(); + let source_vertex_id = edge_wrapper_enum.source().primary_key(); + let target_vertex_id = edge_wrapper_enum.target().primary_key(); + let mut edge_attr_map = edge_wrapper_enum.to_json_value(); + edge_attr_map.insert("from_id".to_string(), json!(source_vertex_id)); + edge_attr_map.insert("to_id".to_string(), json!(target_vertex_id)); + edge_attr_map.insert("edge_type".to_string(), json!(edge_type)); + connections.push(Value::Object(edge_attr_map)) + } + } - let inner_map = vertices_map.entry(outer_map_key).or_insert(HashMap::new()); - inner_map.insert(inner_map_key, edge_wrapper.source.to_attributes_map()); + let edges_str = + serde_json::to_string(&connections).map_err(|err| Error::JSONParseError(err))?; + Ok(ContractEdgesRequest { edges_str }) + } +} + +impl From for UpsertGraph { + fn from(edges: BatchEdges) -> Self { + let mut edges_map = HashMap::new(); + let mut vertices_map: HashMap>> = + HashMap::new(); + + for edge_wrapper_enum in edges.0 { + let source_vertex_type = edge_wrapper_enum.source().vertex_type(); + let source_vertex_id = edge_wrapper_enum.source().primary_key(); + let edge_type = edge_wrapper_enum.e_type(); + let target_vertex_type = edge_wrapper_enum.target().vertex_type(); + let target_vertex_id = edge_wrapper_enum.target().primary_key(); + let edge_attributes = edge_wrapper_enum.to_attributes_map(); + + edges_map + .entry(source_vertex_type.clone()) + .or_insert_with(HashMap::new) + .entry(source_vertex_id.clone()) + .or_insert_with(HashMap::new) + .entry(edge_type.to_string()) + .or_insert_with(HashMap::new) + .entry(target_vertex_type.clone()) + .or_insert_with(HashMap::new) + .insert(target_vertex_id.clone(), edge_attributes); + + // Helper function to merge vertex attributes + fn merge_vertex_attributes( + vertices_map: &mut HashMap>>, + vertex_type: &str, + vertex_id: &str, + new_attributes: HashMap, + ) { + if let Some(existing_attributes) = vertices_map + .entry(vertex_type.to_string()) + .or_insert_with(HashMap::new) + .get_mut(vertex_id) + { + for (key, new_attr) in new_attributes { + match key.as_str() { + "reverse" => { + if let Some(existing_attr) = existing_attributes.get_mut("reverse") + { + if let (Value::Bool(existing_val), Value::Bool(new_val)) = + (&existing_attr.value, new_attr.value) + { + existing_attr.value = json!(*existing_val || new_val); + } + } else { + existing_attributes.insert(key, new_attr); + } + } + "display_name" => { + if let Some(existing_attr) = + existing_attributes.get_mut("display_name") + { + if existing_attr.value == json!("") + && new_attr.value != json!("") + { + existing_attr.value = new_attr.value; + } + } else { + existing_attributes.insert(key, new_attr); + } + } + _ => { + existing_attributes.insert(key, new_attr); + } + } + } + } else { + vertices_map + .get_mut(vertex_type) + .unwrap() + .insert(vertex_id.to_string(), new_attributes); + } } - // Insert target data + // downcast_ref is a method from the Any trait in Rust, + // which allows you to safely attempt to + // convert a reference to a trait object (&dyn Any) + // back into a reference to a specific concrete type (&T) + if let Some(source) = edge_wrapper_enum + .source() + .as_any() + .downcast_ref::() { - let outer_map_key = edge_wrapper.target.vertex_type().clone(); - let inner_map_key = edge_wrapper.target.primary_key().clone(); + merge_vertex_attributes( + &mut vertices_map, + &source_vertex_type, + &source_vertex_id, + source.to_attributes_map(), + ); + } - let inner_map = vertices_map.entry(outer_map_key).or_insert(HashMap::new()); - inner_map.insert(inner_map_key, edge_wrapper.target.to_attributes_map()); + if let Some(target) = edge_wrapper_enum + .target() + .as_any() + .downcast_ref::() + { + merge_vertex_attributes( + &mut vertices_map, + &target_vertex_type, + &target_vertex_id, + target.to_attributes_map(), + ); + } + + if let Some(source) = edge_wrapper_enum + .source() + .as_any() + .downcast_ref::() + { + vertices_map + .entry(source_vertex_type.clone()) + .or_insert_with(HashMap::new) + .insert(source_vertex_id.clone(), source.to_attributes_map()); + } + + if let Some(target) = edge_wrapper_enum + .target() + .as_any() + .downcast_ref::() + { + vertices_map + .entry(target_vertex_type.clone()) + .or_insert_with(HashMap::new) + .insert(target_vertex_id.clone(), target.to_attributes_map()); + } + + if let Some(source) = edge_wrapper_enum + .source() + .as_any() + .downcast_ref::() + { + vertices_map + .entry(source_vertex_type.clone()) + .or_insert_with(HashMap::new) + .insert(source_vertex_id.clone(), source.to_attributes_map()); + } + + if let Some(target) = edge_wrapper_enum + .target() + .as_any() + .downcast_ref::() + { + vertices_map + .entry(target_vertex_type.clone()) + .or_insert_with(HashMap::new) + .insert(target_vertex_id.clone(), target.to_attributes_map()); } } diff --git a/src/tigergraph/upsert.rs b/src/tigergraph/upsert.rs index db776dd2..0f852e54 100644 --- a/src/tigergraph/upsert.rs +++ b/src/tigergraph/upsert.rs @@ -17,6 +17,7 @@ use http::uri::InvalidUri; use hyper::Method; use hyper::{client::HttpConnector, Body, Client}; use serde::{Deserialize, Serialize}; +use serde_json::value::Value; use std::collections::HashMap; use std::convert::{TryFrom, TryInto}; use tracing::{error, trace}; @@ -286,12 +287,12 @@ where let from_record = VertexRecord::from_with_json_value( warpper.source.vertex_type(), warpper.source.primary_key(), - warpper.source.to_json_value(), + Value::Object(warpper.source.to_json_value()), ); let to_record = VertexRecord::from_with_json_value( warpper.target.vertex_type(), warpper.target.primary_key(), - warpper.target.to_json_value(), + Value::Object(warpper.target.to_json_value()), ); let from_str = serde_json::to_string(&from_record).map_err(|err| Error::JSONParseError(err))?; @@ -319,7 +320,7 @@ where let vertex_record = VertexRecord::from_with_json_value( warpper.vertex.vertex_type(), warpper.vertex.primary_key(), - warpper.vertex.to_json_value(), + Value::Object(warpper.vertex.to_json_value()), ); let vertex_str = serde_json::to_string(&vertex_record).map_err(|err| Error::JSONParseError(err))?; @@ -525,6 +526,36 @@ pub async fn create_identity_to_identity_hold_record( Ok(()) } +pub async fn create_ens_identity_resolve( + client: &Client, + ens_identity: &Identity, + evm_address: &Identity, + resolve: &Resolve, +) -> Result<(), Error> { + let resolve_identity = resolve.wrapper(ens_identity, evm_address, RESOLVE); + let edges = Edges(vec![resolve_identity]); + let upsert_edge_req: UpsertEdge = edges.into(); + + // Only create EvmAddress as an Identity connect to HyperVertex in case evm_address owner_address!=resolve_address + let vertex_wrapper = IsolatedVertexWrapper { + vertex: evm_address.to_owned(), + }; + let upsert_isolated_vertex_req: UpsertIsolatedVertex = vertex_wrapper.try_into()?; + + let vertices = Vertices(vec![ens_identity.to_owned()]); + let vertices_map: HashMap>> = + vertices.into(); + let ens_wrapper = UpsertVertices { + vertices: vertices_map, + }; + let upsert_ens_req: UpsertVertices = ens_wrapper.into(); + + upsert_isolated_vertex(&client, &upsert_isolated_vertex_req, Graph::SocialGraph).await?; + upsert_vertices(client, &upsert_ens_req, Graph::SocialGraph).await?; + upsert_edge(&client, &upsert_edge_req, Graph::SocialGraph).await?; + Ok(()) +} + pub async fn create_ens_identity_ownership( client: &Client, evm_address: &Identity, diff --git a/src/tigergraph/vertex/contract.rs b/src/tigergraph/vertex/contract.rs index 50238e5b..6b24d23b 100644 --- a/src/tigergraph/vertex/contract.rs +++ b/src/tigergraph/vertex/contract.rs @@ -18,6 +18,7 @@ use hyper::{client::HttpConnector, Body, Client, Method}; use serde::{Deserialize, Serialize}; use serde_json::json; use serde_json::value::{Map, Value}; +use std::any::Any; use std::collections::HashMap; use tracing::{error, trace}; use uuid::Uuid; @@ -72,6 +73,10 @@ impl Vertex for Contract { fn vertex_type(&self) -> String { VERTEX_NAME.to_string() } + + fn as_any(&self) -> &dyn Any { + self + } } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -183,7 +188,7 @@ impl Transfer for Contract { attributes_map } - fn to_json_value(&self) -> Value { + fn to_json_value(&self) -> Map { let mut map = Map::new(); map.insert("id".to_string(), json!(self.primary_key())); map.insert("uuid".to_string(), json!(self.uuid)); @@ -195,7 +200,7 @@ impl Transfer for Contract { json!(self.symbol.clone().unwrap_or("".to_string())), ); map.insert("updated_at".to_string(), json!(self.updated_at)); - Value::Object(map) + map } } diff --git a/src/tigergraph/vertex/identity.rs b/src/tigergraph/vertex/identity.rs index f55bc045..97e5c290 100644 --- a/src/tigergraph/vertex/identity.rs +++ b/src/tigergraph/vertex/identity.rs @@ -28,6 +28,7 @@ use serde::de::{self, Deserializer, MapAccess, Visitor}; use serde::{Deserialize, Serialize}; use serde_json::json; use serde_json::value::{Map, Value}; +use std::any::Any; use std::collections::HashMap; use std::fmt; use tracing::{error, trace}; @@ -91,6 +92,10 @@ impl Vertex for Identity { fn vertex_type(&self) -> String { VERTEX_NAME.to_string() } + + fn as_any(&self) -> &dyn Any { + self + } } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -275,7 +280,7 @@ impl Transfer for Identity { attributes_map } - fn to_json_value(&self) -> Value { + fn to_json_value(&self) -> Map { let mut map = Map::new(); map.insert("id".to_string(), json!(self.primary_key())); map.insert( @@ -316,7 +321,7 @@ impl Transfer for Identity { "reverse".to_string(), self.reverse.map_or(json!(false), |reverse| json!(reverse)), ); - Value::Object(map) + map } } diff --git a/src/tigergraph/vertex/identity_graph.rs b/src/tigergraph/vertex/identity_graph.rs index b0a0f91b..1b0c1167 100644 --- a/src/tigergraph/vertex/identity_graph.rs +++ b/src/tigergraph/vertex/identity_graph.rs @@ -2,18 +2,147 @@ use crate::{ config::C, error::Error, tigergraph::{ - vertex::{Identity, IdentityRecord, VertexRecord}, - BaseResponse, Graph, + vertex::{FromWithParams, Identity, IdentityRecord, Vertex, VertexRecord}, + Attribute, BaseResponse, Graph, OpCode, Transfer, }, upstream::{Chain, DataSource, Platform}, util::parse_body, }; +use async_trait::async_trait; use http::uri::InvalidUri; use hyper::{client::HttpConnector, Body, Client, Method}; use serde::de::{self, MapAccess, Visitor}; use serde::{Deserialize, Serialize}; +use serde_json::json; +use serde_json::value::{Map, Value}; +use std::any::Any; +use std::collections::HashMap; use tracing::error; +pub const VERTEX_NAME: &str = "IdentitiesGraph"; + +/// IdentitiesGraph +#[derive(Clone, Serialize, Deserialize, Debug)] +pub struct IdentitiesGraph { + /// UUID of this record + pub id: String, + /// microseconds are one-millionth of a second (1/1,000,000 seconds) + pub updated_nanosecond: i64, +} + +impl Default for IdentitiesGraph { + fn default() -> Self { + Self { + id: String::from("fake_uuid_v4"), + updated_nanosecond: Default::default(), + } + } +} + +impl PartialEq for IdentitiesGraph { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +// #[typetag::serde] +#[async_trait] +impl Vertex for IdentitiesGraph { + fn primary_key(&self) -> String { + self.id.clone() + } + + fn vertex_type(&self) -> String { + VERTEX_NAME.to_string() + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct IdentitiesGraphRecord(pub VertexRecord); + +impl FromWithParams for IdentitiesGraphRecord { + fn from_with_params(v_type: String, v_id: String, attributes: IdentitiesGraph) -> Self { + IdentitiesGraphRecord(VertexRecord { + v_type, + v_id, + attributes, + }) + } +} + +impl From> for IdentitiesGraphRecord { + fn from(record: VertexRecord) -> Self { + IdentitiesGraphRecord(record) + } +} + +impl std::ops::Deref for IdentitiesGraphRecord { + type Target = VertexRecord; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for IdentitiesGraphRecord { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl std::ops::Deref for VertexRecord { + type Target = IdentitiesGraph; + + fn deref(&self) -> &Self::Target { + &self.attributes + } +} + +impl std::ops::DerefMut for VertexRecord { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.attributes + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct IdentitiesGraphAttribute(HashMap); + +// Implement `Transfer` trait for converting `IdentitiesGraph` into a `HashMap`. +impl Transfer for IdentitiesGraph { + fn to_attributes_map(&self) -> HashMap { + let mut attributes_map = HashMap::new(); + attributes_map.insert( + "id".to_string(), + Attribute { + value: json!(self.id), + op: Some(OpCode::IgnoreIfExists), + }, + ); + attributes_map.insert( + "updated_nanosecond".to_string(), + Attribute { + value: json!(self.updated_nanosecond), + op: Some(OpCode::IgnoreIfExists), + }, + ); + attributes_map + } + + fn to_json_value(&self) -> Map { + let mut map = Map::new(); + map.insert("id".to_string(), json!(self.id)); + map.insert( + "updated_nanosecond".to_string(), + json!(self.updated_nanosecond), + ); + map + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct IdentityConnection { pub edge_type: String, diff --git a/src/tigergraph/vertex/mod.rs b/src/tigergraph/vertex/mod.rs index 6c7955ea..aee59f8e 100644 --- a/src/tigergraph/vertex/mod.rs +++ b/src/tigergraph/vertex/mod.rs @@ -7,9 +7,12 @@ pub use identity::{ ExpireTimeLoadFn, Identity, IdentityLoadFn, IdentityRecord, IdentityWithSource, NeighborReverseLoadFn, NeighborsResponse, OwnerLoadFn, }; -pub use identity_graph::{Address, ExpandIdentityRecord, IdentityConnection, IdentityGraph}; +pub use identity_graph::{ + Address, ExpandIdentityRecord, IdentitiesGraph, IdentityConnection, IdentityGraph, +}; use serde::{Deserialize, Serialize}; use serde_json::value::Value; +use std::any::Any; /// All `Vertex` records. #[async_trait] @@ -17,6 +20,8 @@ pub trait Vertex { fn primary_key(&self) -> String; fn vertex_type(&self) -> String; + + fn as_any(&self) -> &dyn Any; } #[derive(Debug, Clone, Deserialize, Serialize)] diff --git a/src/upstream/aggregation/mod.rs b/src/upstream/aggregation/mod.rs index 56eba27a..0023ca50 100644 --- a/src/upstream/aggregation/mod.rs +++ b/src/upstream/aggregation/mod.rs @@ -6,6 +6,7 @@ use crate::error::Error; use crate::tigergraph::edge::Proof; use crate::tigergraph::upsert::create_identity_to_identity_proof_two_way_binding; use crate::tigergraph::vertex::Identity; +use crate::tigergraph::EdgeList; use crate::upstream::{DataSource, Fetcher, Platform, ProofLevel, TargetProcessedList}; use crate::util::{ make_client, make_http_client, naive_now, parse_body, request_with_timeout, timestamp_to_naive, @@ -61,6 +62,13 @@ impl Fetcher for Aggregation { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + Ok((vec![], vec![])) + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::Ethereum, Platform::Twitter]) } diff --git a/src/upstream/crossbell/mod.rs b/src/upstream/crossbell/mod.rs index ae68032e..22fedbc6 100644 --- a/src/upstream/crossbell/mod.rs +++ b/src/upstream/crossbell/mod.rs @@ -3,11 +3,13 @@ mod tests; use crate::config::C; use crate::error::Error; -use crate::tigergraph::edge::{Hold, Resolve}; +use crate::tigergraph::edge::{Hold, HyperEdge, Resolve, Wrapper}; +use crate::tigergraph::edge::{HOLD_IDENTITY, HYPER_EDGE, RESOLVE, REVERSE_RESOLVE}; use crate::tigergraph::upsert::create_identity_domain_resolve_record; use crate::tigergraph::upsert::create_identity_domain_reverse_resolve_record; use crate::tigergraph::upsert::create_identity_to_identity_hold_record; -use crate::tigergraph::vertex::Identity; +use crate::tigergraph::vertex::{IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; use crate::upstream::{ DataFetcher, DataSource, DomainNameSystem, Fetcher, Platform, Target, TargetProcessedList, }; @@ -115,11 +117,205 @@ impl Fetcher for Crossbell { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + match target.platform()? { + Platform::Ethereum => batch_fetch_by_wallet(target).await, + Platform::Crossbell => batch_fetch_by_handle(target).await, + _ => Ok((vec![], vec![])), + } + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::Ethereum, Platform::Crossbell]) } } +async fn query_by_handle(target: &Target) -> Result, Error> { + let query = QUERY_BY_HANDLE.to_string(); + let target_var = target.identity()?; + let handle = target_var.trim_end_matches(".csb"); + let client = GQLClient::new(&C.upstream.crossbell_api.url); + let vars = QueryVars { + target: handle.to_string(), + }; + let resp = client.query_with_vars::(&query, vars); + + let data: Option = + match tokio::time::timeout(std::time::Duration::from_secs(5), resp).await { + Ok(resp) => match resp { + Ok(resp) => resp, + Err(err) => { + warn!(?target, ?err, "Crossbell: Failed to fetch"); + None + } + }, + Err(_) => { + warn!(?target, "Crossbell timeout: no response in 5 seconds."); + None + } + }; + + Ok(data) +} + +async fn query_by_wallet(target: &Target) -> Result, Error> { + let query = QUERY_BY_WALLET.to_string(); + let target_var = target.identity()?; + let client = GQLClient::new(&C.upstream.crossbell_api.url); + let vars = QueryVars { + target: target_var.to_lowercase(), + }; + let resp = client.query_with_vars::(&query, vars); + + let data: Option = + match tokio::time::timeout(std::time::Duration::from_secs(5), resp).await { + Ok(resp) => match resp { + Ok(resp) => resp, + Err(err) => { + warn!(?target, ?err, "Crossbell: Failed to fetch"); + None + } + }, + Err(_) => { + warn!(?target, "Crossbell timeout: no response in 5 seconds."); + None + } + }; + + Ok(data) +} + +async fn batch_fetch_by_handle(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let data = query_by_handle(target).await?; + if data.is_none() { + info!(?target, "Crossbell: No result"); + return Ok((vec![], vec![])); + } + let res = data.unwrap(); + debug!(?target, characters = res.characters.len(), "Records found."); + + let owner = res.characters.first().unwrap().owner.clone().to_lowercase(); + let mut next_targets = TargetProcessedList::new(); + next_targets.push(Target::Identity(Platform::Ethereum, owner)); + + let edges = generate_edges(&res.characters); + Ok((next_targets, edges)) +} + +async fn batch_fetch_by_wallet(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let data = query_by_wallet(target).await?; + if data.is_none() { + info!(?target, "Crossbell: No result"); + return Ok((vec![], vec![])); + } + let res = data.unwrap(); + debug!(?target, characters = res.characters.len(), "Records found."); + let edges = generate_edges(&res.characters); + // after fetch by wallet, nothing return for next target + Ok((vec![], edges)) +} + +fn generate_edges(characters: &Vec) -> EdgeList { + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + for profile in characters.iter() { + let handle = profile.handle.clone(); + let csb = format!("{}.csb", handle); + let display_name = profile.metadata.clone().map_or(handle.clone(), |res| { + res.content.map_or(handle.clone(), |content| { + content.name.map_or(handle.clone(), |name| name) + }) + }); + let avatar = profile.metadata.clone().map_or(None, |res| { + res.content.map_or(None, |content| { + content + .avatars + .map_or(None, |avatars| avatars.first().cloned()) + }) + }); + + let mut crossbell = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Crossbell, + identity: csb.clone(), + uid: Some(profile.character_id.clone()), + created_at: profile.created_at, + display_name: Some(display_name), + added_at: naive_now(), + avatar_url: avatar, + profile_url: Some("https://xchar.app/".to_owned() + &profile.handle.clone()), + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let owner = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Ethereum, + identity: profile.owner.clone(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::Crossbell, + transaction: profile.transaction_hash.clone(), + id: profile.character_id.clone(), + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: None, + }; + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::Crossbell, + system: DomainNameSystem::Crossbell, + name: csb.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + + if profile.primary { + let reverse: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::Crossbell, + system: DomainNameSystem::Crossbell, + name: csb.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + crossbell.reverse = Some(true); + let rrs = reverse.wrapper(&owner, &crossbell, REVERSE_RESOLVE); + edges.push(EdgeWrapperEnum::new_reverse_resolve(rrs)); + } + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &crossbell, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &owner, HYPER_EDGE), + )); + + let hd = hold.wrapper(&owner, &crossbell, HOLD_IDENTITY); + let rs = resolve.wrapper(&crossbell, &owner, RESOLVE); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + } + + edges +} + async fn fetch_by_wallet(target: &Target) -> Result { let query = QUERY_BY_WALLET.to_string(); let target_var = target.identity()?; diff --git a/src/upstream/dotbit/mod.rs b/src/upstream/dotbit/mod.rs index 01401c13..f98df690 100644 --- a/src/upstream/dotbit/mod.rs +++ b/src/upstream/dotbit/mod.rs @@ -2,11 +2,13 @@ mod tests; use crate::config::C; use crate::error::Error; -use crate::tigergraph::edge::{Hold, Resolve}; +use crate::tigergraph::edge::{Hold, HyperEdge, Resolve, Wrapper}; +use crate::tigergraph::edge::{HOLD_IDENTITY, HYPER_EDGE, RESOLVE, REVERSE_RESOLVE}; use crate::tigergraph::upsert::create_identity_domain_resolve_record; use crate::tigergraph::upsert::create_identity_domain_reverse_resolve_record; use crate::tigergraph::upsert::create_identity_to_identity_hold_record; -use crate::tigergraph::vertex::Identity; +use crate::tigergraph::vertex::{IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; use crate::upstream::{ DataFetcher, DataSource, DomainNameSystem, Fetcher, Platform, Target, TargetProcessedList, }; @@ -38,6 +40,22 @@ impl Fetcher for DotBit { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + match target.platform()? { + Platform::Dotbit => batch_fetch_by_handle(target).await, + Platform::Ethereum => batch_fetch_by_wallet(target).await, + Platform::CKB => batch_fetch_by_wallet(target).await, + Platform::Tron => batch_fetch_by_wallet(target).await, + Platform::Polygon => batch_fetch_by_wallet(target).await, + Platform::BNBSmartChain => batch_fetch_by_wallet(target).await, + _ => Ok((vec![], vec![])), + } + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![ Platform::Dotbit, @@ -51,12 +69,12 @@ impl Fetcher for DotBit { } /// API docs https://github.com/dotbitHQ/das-account-indexer/blob/main/API.md -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct AccInfoRequestParams { pub account: String, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct AccInfoRequest { pub jsonrpc: String, pub id: i32, @@ -64,13 +82,13 @@ pub struct AccInfoRequest { pub params: Vec, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct OutPoint { pub tx_hash: String, pub index: i64, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct AccountInfo { pub account: String, pub account_alias: Option, @@ -82,41 +100,41 @@ pub struct AccountInfo { pub display_name: String, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct AccountInfoData { pub out_point: Option, pub account_info: Option, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct AccountInfoResult { pub errno: Option, pub errmsg: Option, pub data: Option, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct AccountInfoResponse { pub id: Option, pub jsonrpc: String, pub result: AccountInfoResult, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct RequestKeyInfo { pub coin_type: String, pub chain_id: String, pub key: String, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct RequestTypeKeyInfoParams { #[serde(rename = "type")] pub req_type: String, pub key_info: RequestKeyInfo, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct ReverseRecordRequest { pub jsonrpc: String, pub id: i32, @@ -124,7 +142,7 @@ pub struct ReverseRecordRequest { pub params: Vec, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct AccountItem { pub account: String, pub account_alias: Option, @@ -132,33 +150,33 @@ pub struct AccountItem { pub expired_at: Option, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct ReverseResult { pub errno: Option, pub errmsg: Option, pub data: Option, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct ReverseResponse { pub id: Option, pub jsonrpc: String, pub result: ReverseResult, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct AccountListData { pub account_list: Vec, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct AccountListResult { pub errno: Option, pub errmsg: Option, pub data: Option, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct AccountListResponse { pub id: Option, pub jsonrpc: String, @@ -167,6 +185,348 @@ pub struct AccountListResponse { const UNKNOWN_OWNER: &str = "0x0000000000000000000000000000000000000000"; +async fn query_by_handle(_platform: &Platform, name: &str) -> Result { + let request_acc = AccInfoRequestParams { + account: name.to_string(), + }; + let params = AccInfoRequest { + jsonrpc: "2.0".to_string(), + id: 1, + method: "das_accountInfo".to_string(), + params: vec![request_acc], + }; + let json_params = serde_json::to_vec(¶ms)?; + + let client = make_client(); + let req = Request::builder() + .method(Method::POST) + .uri(C.upstream.dotbit_service.url.clone()) + .body(Body::from(json_params)) + .map_err(|_err| Error::ParamError(format!("Dotbit Build Request Error {}", _err)))?; + + let mut result = request_with_timeout(&client, req, Some(std::time::Duration::from_secs(30))) + .await + .map_err(|err| { + Error::ManualHttpClientError(format!( + "Dotbit fetch | das_accountInfo error: {:?}", + err.to_string() + )) + })?; + + let resp: AccountInfoResponse = parse_body(&mut result).await?; + if resp.result.errno.map_or(false, |e| e != 0) { + warn!("fail to fetch the result from .bit, resp {:?}", resp); + return Err(Error::NoResult); + } + + let info = resp.result.data.unwrap(); + let account_info = info.clone().account_info.unwrap(); + // tricky way to remove the unexpected case... + // will be removed after confirmied with .bit team how to define its a .bit NFT on Ethereum + // https://talk.did.id/t/convert-your-bit-to-nft-on-ethereum-now/481 + if account_info.owner_key == UNKNOWN_OWNER { + warn!(".bit profile owner is zero address"); + return Err(Error::NoResult); + } + + let account_platform: Platform = account_info.owner_algorithm_id.into(); + if account_platform == Platform::Unknown { + let warn_message = format!( + ".bit profile owner_algorithm_id(value={}) map to platform is Unknown", + account_info.owner_algorithm_id + ); + warn!(warn_message); + return Err(Error::ParamError(warn_message)); + } + + Ok(info) +} + +async fn query_by_wallet(platform: &Platform, address: &str) -> Result, Error> { + // das_accountList + let coin_type: CoinType = platform.clone().into(); + if coin_type == CoinType::Unknown { + return Ok(vec![]); + } + let request_params = get_req_params(&coin_type, address); + let params = ReverseRecordRequest { + jsonrpc: "2.0".to_string(), + id: 1, + method: "das_accountList".to_string(), + params: vec![request_params], + }; + let json_params = serde_json::to_vec(¶ms)?; + + let client = make_client(); + let req = Request::builder() + .method(Method::POST) + .uri(C.upstream.dotbit_service.url.clone()) + .body(Body::from(json_params)) + .map_err(|_err| Error::ParamError(format!("Dotbit Build Request Error {}", _err)))?; + + let mut result = request_with_timeout(&client, req, None) + .await + .map_err(|err| { + Error::ManualHttpClientError(format!( + "Dotbit fetch | das_accountList error: {:?}", + err.to_string() + )) + })?; + + let resp: AccountListResponse = parse_body(&mut result).await?; + if resp.result.errno.map_or(false, |e| e != 0) { + warn!("fail to fetch the result from .bit, resp {:?}", resp); + return Err(Error::NoResult); + } + if resp.result.data.is_none() { + warn!("fail to fetch the result from .bit, resp {:?}", resp); + return Err(Error::NoResult); + } + + Ok(resp.result.data.unwrap().account_list) +} + +async fn query_reverse_record(platform: &Platform, identity: &str) -> Result { + let coin_type: CoinType = platform.clone().into(); + if coin_type == CoinType::Unknown { + return Err(Error::ParamError(format!( + "platform({}) convert to dotbit coin_type: unknown", + platform.to_string() + ))); + } + // fetch addr's reverse record: das_reverseRecord + let request_params = get_req_params(&coin_type, identity); + let params = ReverseRecordRequest { + jsonrpc: "2.0".to_string(), + id: 1, + method: "das_reverseRecord".to_string(), + params: vec![request_params], + }; + let json_params = serde_json::to_vec(¶ms)?; + + let client = make_client(); + let req = Request::builder() + .method(Method::POST) + .uri(C.upstream.dotbit_service.url.clone()) + .body(Body::from(json_params)) + .map_err(|_err| Error::ParamError(format!("Dotbit Build Request Error {}", _err)))?; + + let mut result = request_with_timeout(&client, req, None) + .await + .map_err(|err| { + Error::ManualHttpClientError(format!( + "Dotbit fetch | das_reverseRecord error: {:?}", + err.to_string() + )) + })?; + let resp: ReverseResponse = parse_body(&mut result).await?; + if resp.result.errno.map_or(false, |e| e != 0) { + warn!("fail to fetch the result from .bit, resp {:?}", resp); + return Err(Error::NoResult); + } + if resp.result.data.is_none() || resp.result.data.as_ref().unwrap().account.len() == 0 { + warn!("das_reverseRecord result is empty, resp {:?}", resp); + return Err(Error::NoResult); + } + + Ok(resp.result.data.unwrap()) +} + +async fn batch_fetch_by_wallet(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let platform = target.platform()?; + let identity = target.identity()?; + + // fetch addr's hold dotbit records + let result = query_by_wallet(&platform, &identity).await?; + // fetch addr's reverse record + + let default_dotbit = match query_reverse_record(&platform, &identity).await { + Ok(record) => record.account, + Err(_) => "".to_string(), + }; + let reverse_flag = !default_dotbit.is_empty(); // if default_dotbit is an empty string, reverse = false + + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + let mut wallet: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: platform.clone(), + identity: identity.to_string().to_lowercase().clone(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(reverse_flag), + }; + + for i in result.into_iter() { + let create_at_naive = option_timestamp_to_naive(i.registered_at, 0); + let expired_at_naive = option_timestamp_to_naive(i.expired_at, 0); + let domain_name = i.account.to_string(); + let mut dotbit: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Dotbit, + identity: domain_name.clone(), + uid: None, + created_at: create_at_naive, + display_name: Some(domain_name.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: expired_at_naive, + reverse: Some(false), + }; + + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::Dotbit, + transaction: None, + id: "".to_string(), + created_at: create_at_naive, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: expired_at_naive, + }; + + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::Dotbit, + system: DomainNameSystem::DotBit, + name: domain_name.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + + if reverse_flag && domain_name == default_dotbit { + let reverse: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::Dotbit, + system: DomainNameSystem::DotBit, + name: domain_name.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + dotbit.reverse = Some(true); + wallet.reverse = Some(true); + let rrs = reverse.wrapper(&wallet, &dotbit, REVERSE_RESOLVE); + edges.push(EdgeWrapperEnum::new_reverse_resolve(rrs)); + } + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &wallet, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &dotbit, HYPER_EDGE), + )); + + // hold record + let hd = hold.wrapper(&wallet, &dotbit, HOLD_IDENTITY); + // 'regular' resolution involves mapping from a name to an address. + let rs = resolve.wrapper(&dotbit, &wallet, RESOLVE); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + } + + // after fetch by wallet, nothing return for next target + Ok((vec![], edges)) +} + +async fn batch_fetch_by_handle(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let platform = target.platform()?; + let identity = target.identity()?; + + let info = query_by_handle(&platform, &identity).await?; + let account_info = info.account_info.unwrap(); + let out_point = info.out_point.unwrap(); + + let account_addr = account_info.owner_key.to_lowercase(); + let account_platform: Platform = account_info.owner_algorithm_id.into(); + + let mut next_targets = TargetProcessedList::new(); + next_targets.push(Target::Identity( + account_platform.clone(), + account_addr.clone(), + )); + + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + let created_at_naive = timestamp_to_naive(account_info.create_at_unix, 0); + let expired_at_naive = timestamp_to_naive(account_info.expired_at_unix, 0); + + let wallet: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: account_platform, + identity: account_addr.clone(), + uid: None, + created_at: created_at_naive, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let dotbit: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Dotbit, + identity: identity.to_string(), + uid: None, + created_at: created_at_naive, + display_name: Some(identity.to_string()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: expired_at_naive, + reverse: Some(false), + }; + + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::Dotbit, + transaction: Some(out_point.tx_hash), + id: out_point.index.to_string(), + created_at: created_at_naive, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: expired_at_naive, + }; + + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::Dotbit, + system: DomainNameSystem::DotBit, + name: identity.to_string(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &wallet, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &dotbit, HYPER_EDGE), + )); + + // hold record + let hd = hold.wrapper(&wallet, &dotbit, HOLD_IDENTITY); + // 'regular' resolution involves mapping from a name to an address. + let rs = resolve.wrapper(&dotbit, &wallet, RESOLVE); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + + Ok((next_targets, edges)) +} + async fn fetch_connections_by_platform_identity( platform: &Platform, identity: &str, @@ -388,7 +748,10 @@ async fn fetch_reverse_record( return Err(Error::NoResult); } if resp.result.data.is_none() || resp.result.data.as_ref().unwrap().account.len() == 0 { - warn!("das_reverseRecord result is empty, resp {:?}", resp); + warn!( + "das_reverseRecord result({}) is empty, resp {:?}", + identity, resp + ); return Ok(None); } diff --git a/src/upstream/ens_reverse/mod.rs b/src/upstream/ens_reverse/mod.rs index af49e5ba..8e783789 100644 --- a/src/upstream/ens_reverse/mod.rs +++ b/src/upstream/ens_reverse/mod.rs @@ -3,11 +3,14 @@ mod tests; use crate::config::C; use crate::error::Error; -use crate::tigergraph::edge::Resolve; +use crate::tigergraph::edge::{ + HyperEdge, Resolve, Wrapper, HYPER_EDGE, REVERSE_RESOLVE, REVERSE_RESOLVE_CONTRACT, +}; use crate::tigergraph::upsert::create_identity_domain_reverse_resolve_record; use crate::tigergraph::upsert::create_identity_to_contract_reverse_resolve_record; use crate::tigergraph::upsert::create_isolated_vertex; -use crate::tigergraph::vertex::{Contract, Identity}; +use crate::tigergraph::vertex::{Contract, IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; use crate::upstream::{Chain, ContractCategory, DataFetcher, DataSource, DomainNameSystem}; use crate::util::{make_client, make_http_client, naive_now, parse_body, request_with_timeout}; use async_trait::async_trait; @@ -98,6 +101,89 @@ impl Fetcher for ENSReverseLookup { Ok(vec![]) } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + let wallet = target.identity().unwrap().to_lowercase(); + let record = fetch_record(&wallet).await?; + // If reverse lookup record is reset to empty by user, + // our cache should also be cleared. + // Reach this by setting `display_name` into `Some("")`. + let reverse_ens = record.reverse_record.clone().unwrap_or("".into()); + let mut eth_identity = Identity::default(); + eth_identity.uuid = Some(Uuid::new_v4()); + eth_identity.platform = Platform::Ethereum; + eth_identity.identity = wallet.clone(); + eth_identity.display_name = Some(reverse_ens.clone()); + + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + if reverse_ens == "" { + // if ens reverse is empty, we should also saving the ethereum into identity_graph, create a isolated vertex + edges.push(EdgeWrapperEnum::new_hyper_edge(HyperEdge {}.wrapper( + &hv, + ð_identity, + HYPER_EDGE, + ))); + info!(?target, "ENS Reverse record is null"); + return Ok((vec![], edges)); + } + info!(?target, "ENS Reverse record: {} => {}", wallet, reverse_ens); + + eth_identity.reverse = Some(true); // ethereum and primary ens remain same value + let ens_domain = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::ENS, + identity: reverse_ens.clone(), + uid: None, + created_at: None, + display_name: Some(reverse_ens.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(true), + }; + let contract = Contract { + uuid: Uuid::new_v4(), + category: ContractCategory::ENS, + address: ContractCategory::ENS.default_contract_address().unwrap(), + chain: Chain::Ethereum, + symbol: None, + updated_at: naive_now(), + }; + + let reverse = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::TheGraph, + system: DomainNameSystem::ENS, + name: reverse_ens.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + // create reverse resolve record + let rr = reverse.wrapper(ð_identity, &ens_domain, REVERSE_RESOLVE); + let rrc = reverse.wrapper(ð_identity, &contract, REVERSE_RESOLVE_CONTRACT); + edges.push(EdgeWrapperEnum::new_hyper_edge(HyperEdge {}.wrapper( + &hv, + ð_identity, + HYPER_EDGE, + ))); + edges.push(EdgeWrapperEnum::new_hyper_edge(HyperEdge {}.wrapper( + &hv, + &ens_domain, + HYPER_EDGE, + ))); + edges.push(EdgeWrapperEnum::new_reverse_resolve(rr)); + edges.push(EdgeWrapperEnum::new_reverse_resolve_contract(rrc)); + + Ok((vec![], edges)) + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::Ethereum]) } diff --git a/src/upstream/farcaster/mod.rs b/src/upstream/farcaster/mod.rs index 9a3140fb..2e97f83a 100644 --- a/src/upstream/farcaster/mod.rs +++ b/src/upstream/farcaster/mod.rs @@ -5,6 +5,7 @@ use crate::error::Error; use crate::tigergraph::edge::Hold; use crate::tigergraph::upsert::create_identity_to_identity_hold_record; use crate::tigergraph::vertex::Identity; +use crate::tigergraph::EdgeList; use crate::upstream::{DataFetcher, DataSource, Fetcher, Platform, Target, TargetProcessedList}; use crate::util::{make_http_client, naive_now}; use async_trait::async_trait; @@ -30,6 +31,19 @@ impl Fetcher for Farcaster { Target::NFT(_, _, _, _) => todo!(), } } + + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + match target { + Target::Identity(platform, identity) => { + warpcast::batch_fetch_connections(platform, identity).await + } + Target::NFT(_, _, _, _) => todo!(), + } + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::Farcaster, Platform::Ethereum]) } diff --git a/src/upstream/farcaster/tests.rs b/src/upstream/farcaster/tests.rs index 69853fb9..9d502498 100644 --- a/src/upstream/farcaster/tests.rs +++ b/src/upstream/farcaster/tests.rs @@ -1,13 +1,13 @@ #[cfg(test)] mod tests { use crate::error::Error; - use crate::upstream::farcaster::warpcast::{fetch_by_signer, fetch_by_username}; + use crate::upstream::farcaster::warpcast::{batch_fetch_by_signer, batch_fetch_by_username}; use crate::upstream::types::Platform; #[tokio::test] async fn test_get_farcaster_profile_by_username() -> Result<(), Error> { let username = "suji"; - let data = fetch_by_username(&Platform::Farcaster, &username).await?; + let data = batch_fetch_by_username(&Platform::Farcaster, &username).await?; println!("data: {:?}", data); Ok(()) } @@ -15,7 +15,7 @@ mod tests { #[tokio::test] async fn test_get_farcaster_profile_by_signer() -> Result<(), Error> { let address = "0x934b510d4c9103e6a87aef13b816fb080286d649"; - let data = fetch_by_signer(&Platform::Farcaster, &address).await?; + let data = batch_fetch_by_signer(&Platform::Farcaster, &address).await?; println!("data: {:?}", data); Ok(()) } diff --git a/src/upstream/farcaster/warpcast.rs b/src/upstream/farcaster/warpcast.rs index 7e7226de..6da7922a 100644 --- a/src/upstream/farcaster/warpcast.rs +++ b/src/upstream/farcaster/warpcast.rs @@ -2,7 +2,13 @@ use crate::{ config::C, error::Error, tigergraph::upsert::{create_identity_to_identity_hold_record, create_isolated_vertex}, - tigergraph::{edge::Hold, vertex::Identity}, + tigergraph::{ + EdgeList, EdgeWrapperEnum, + { + edge::{Hold, HyperEdge, Wrapper, HOLD_IDENTITY, HYPER_EDGE}, + vertex::{IdentitiesGraph, Identity}, + }, + }, upstream::{DataFetcher, DataSource, Platform, Target, TargetProcessedList}, util::{ make_client, make_http_client, naive_datetime_from_milliseconds, @@ -28,14 +34,227 @@ pub async fn fetch_connections_by_platform_identity( } } +pub async fn batch_fetch_connections( + platform: &Platform, + identity: &str, +) -> Result<(TargetProcessedList, EdgeList), Error> { + match *platform { + Platform::Farcaster => batch_fetch_by_username(platform, identity).await, + Platform::Ethereum => batch_fetch_by_signer(platform, identity).await, + _ => Ok((vec![], vec![])), + } +} + +pub async fn batch_fetch_by_username( + _platform: &Platform, + username: &str, +) -> Result<(TargetProcessedList, EdgeList), Error> { + let mut targets: Vec = Vec::new(); + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + let user = user_by_username(username).await?; + if user.is_none() { + return Ok((vec![], vec![])); + } + let user = user.unwrap(); + let fid = user.fid; + let verifications = get_verifications(fid).await?; + if verifications.is_none() { + return Ok((vec![], vec![])); + } + let verifications = verifications.unwrap(); + // isolated vertex + if verifications.is_empty() { + let isolated_farcaster: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Farcaster, + identity: user.username.clone(), + uid: Some(user.fid.to_string()), + created_at: None, + display_name: Some(user.display_name.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + edges.push(EdgeWrapperEnum::new_hyper_edge(HyperEdge {}.wrapper( + &hv, + &isolated_farcaster, + HYPER_EDGE, + ))); + return Ok((vec![], edges)); + } + + for verification in verifications.iter() { + let protocol: Platform = verification.protocol.parse()?; + let mut address = verification.address.clone(); + if protocol == Platform::Ethereum { + address = address.to_lowercase(); + } + let wallet: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: protocol, + identity: address.clone(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + let farcaster: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Farcaster, + identity: user.username.clone(), + uid: Some(user.fid.to_string()), + created_at: None, + display_name: Some(user.display_name.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::Farcaster, + transaction: None, + id: "".to_string(), + created_at: Some(verification.timestamp), + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: None, + }; + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &wallet, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &farcaster, HYPER_EDGE), + )); + let hd = hold.wrapper(&wallet, &farcaster, HOLD_IDENTITY); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + targets.push(Target::Identity(protocol, address.clone())) + } + + Ok((targets, edges)) +} + +pub async fn batch_fetch_by_signer( + platform: &Platform, + address: &str, +) -> Result<(TargetProcessedList, EdgeList), Error> { + if platform.to_owned() == Platform::Solana { + // WrapcastV2 not supported user-by-verification?address=solana_address format yet. + return Ok((vec![], vec![])); + } + + let user = user_by_verification(address).await?; + if user.is_none() { + return Ok((vec![], vec![])); + } + let user = user.unwrap(); + + let mut targets: Vec = Vec::new(); + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + let fid = user.fid; + let verifications = get_verifications(fid).await?; + if verifications.is_none() { + return Ok((vec![], vec![])); + } + let verifications = verifications.unwrap(); + if verifications.is_empty() { + return Ok((vec![], vec![])); + } + + for verification in verifications.iter() { + let protocol: Platform = verification.protocol.parse()?; + let mut verification_address = verification.address.clone(); + if protocol == Platform::Ethereum { + verification_address = verification_address.to_lowercase(); + } + let wallet: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: protocol, + identity: verification_address.clone(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + let farcaster: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Farcaster, + identity: user.username.clone(), + uid: Some(user.fid.to_string()), + created_at: None, + display_name: Some(user.display_name.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::Farcaster, + transaction: None, + id: "".to_string(), + created_at: Some(verification.timestamp), + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: None, + }; + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &wallet, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &farcaster, HYPER_EDGE), + )); + let hd = hold.wrapper(&wallet, &farcaster, HOLD_IDENTITY); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + + if address != verification_address { + // Do not push the same target repeatedly + targets.push(Target::Identity(protocol, verification_address.clone())) + } + } + + targets.push(Target::Identity(Platform::Farcaster, user.username.clone())); + Ok((targets, edges)) +} + pub async fn fetch_by_username( _platform: &Platform, username: &str, ) -> Result { let cli = make_http_client(); let user = user_by_username(username).await?; + if user.is_none() { + return Ok(vec![]); + } + let user = user.unwrap(); let fid = user.fid; let verifications = get_verifications(fid).await?; + if verifications.is_none() { + return Ok(vec![]); + } + let verifications = verifications.unwrap(); // isolated vertex if verifications.is_empty() { let u: Identity = Identity { @@ -80,6 +299,10 @@ pub async fn fetch_by_signer( let user = user.unwrap(); let fid = user.fid; let verifications = get_verifications(fid).await?; + if verifications.is_none() { + return Ok(vec![]); + } + let verifications = verifications.unwrap(); for verification in verifications.iter() { let target = save_verifications(&cli, &user, verification).await?; targets.push(target); @@ -142,11 +365,6 @@ async fn save_verifications( } // {"errors":[{"message":"No FID associated with username checkyou"}]} -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct WarpcastError { - pub errors: Vec, -} - #[derive(Debug, Clone, Deserialize, Serialize)] pub struct Message { pub message: String, @@ -154,7 +372,8 @@ pub struct Message { #[derive(Debug, Clone, Deserialize, Serialize)] pub struct UserProfileResponse { - pub result: UserProfileResult, + pub errors: Option>, + pub result: Option, } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -204,7 +423,8 @@ pub struct Location { #[derive(Debug, Clone, Deserialize, Serialize)] pub struct VerificationResponse { - pub result: VerificationResult, + pub errors: Option>, + pub result: Option, } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -222,7 +442,7 @@ pub struct Verification { pub protocol: String, } -async fn user_by_username(username: &str) -> Result { +async fn user_by_username(username: &str) -> Result, Error> { let client = make_client(); let uri: http::Uri = format!( "{}/v2/user-by-username?username={}", @@ -262,19 +482,29 @@ async fn user_by_username(username: &str) -> Result { })?; let result = match parse_body::(&mut resp).await { - Ok(r) => r, - Err(_) => { - let w_err = parse_body::(&mut resp).await?; - let err_message = format!( - "Warpcast fetch error| failed to fetch user-by-username?username={}, message: {:?}", - username, w_err - ); - error!(err_message); - return Err(Error::ManualHttpClientError(err_message)); + Ok(r) => match r.errors { + Some(errors) => { + let err_message = format!( + "Warpcast fetch error| failed to fetch user-by-username?username={}, message: {:?}", + username, errors + ); + error!(err_message); + None + } + None => match r.result { + None => None, + Some(res) => Some(res.user), + }, + }, + Err(err) => { + return Err(Error::ManualHttpClientError(format!( + "Warpcast fetch error | parse_body error: {}", + err + ))); } }; - Ok(result.result.user) + Ok(result) } async fn user_by_verification(address: &str) -> Result, Error> { @@ -329,21 +559,31 @@ async fn user_by_verification(address: &str) -> Result, Error> { })?; let result = match parse_body::(&mut resp).await { - Ok(r) => r, - Err(_) => { - let w_err = parse_body::(&mut resp).await?; - let err_message = format!( - "Warpcast fetch error| failed to fetch user-by-verification?address={}, message: {:?}", - address, w_err - ); - error!(err_message); - return Err(Error::ManualHttpClientError(err_message)); + Ok(r) => match r.errors { + Some(errors) => { + let err_message = format!( + "Warpcast fetch error| failed to fetch user-by-verification?address={}, message: {:?}", + address, errors + ); + error!(err_message); + None + } + None => match r.result { + None => None, + Some(res) => Some(res.user), + }, + }, + Err(err) => { + return Err(Error::ManualHttpClientError(format!( + "Warpcast fetch error | parse_body error: {}", + err + ))); } }; - Ok(Some(result.result.user)) + Ok(result) } -async fn get_verifications(fid: i64) -> Result, Error> { +async fn get_verifications(fid: i64) -> Result>, Error> { let client = make_client(); let uri: http::Uri = format!( "{}/v2/verifications?fid={}", @@ -383,16 +623,26 @@ async fn get_verifications(fid: i64) -> Result, Error> { })?; let result = match parse_body::(&mut resp).await { - Ok(r) => r, - Err(_) => { - let w_err = parse_body::(&mut resp).await?; - let err_message = format!( - "Warpcast fetch error| failed to fetch verifications?fid={}, message: {:?}", - fid, w_err - ); - error!(err_message); - return Err(Error::ManualHttpClientError(err_message)); + Ok(r) => match r.errors { + Some(errors) => { + let err_message = format!( + "Warpcast fetch error| failed to fetch verifications?fid={}, message: {:?}", + fid, errors + ); + error!(err_message); + None + } + None => match r.result { + None => None, + Some(res) => Some(res.verifications), + }, + }, + Err(err) => { + return Err(Error::ManualHttpClientError(format!( + "Warpcast fetch error | parse_body error: {}", + err + ))); } }; - Ok(result.result.verifications) + Ok(result) } diff --git a/src/upstream/firefly/mod.rs b/src/upstream/firefly/mod.rs new file mode 100644 index 00000000..4cb6fe6a --- /dev/null +++ b/src/upstream/firefly/mod.rs @@ -0,0 +1,245 @@ +#[cfg(test)] +mod tests; +use crate::config::C; +use crate::error::Error; +use crate::tigergraph::edge::{HyperEdge, Proof, Wrapper}; +use crate::tigergraph::edge::{HYPER_EDGE, PROOF_EDGE, PROOF_REVERSE_EDGE}; +use crate::tigergraph::vertex::{IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; +use crate::upstream::{ + DataFetcher, DataSource, Fetcher, Platform, ProofLevel, TargetProcessedList, +}; +use crate::util::{make_client, naive_now, parse_body, request_with_timeout, timestamp_to_naive}; + +use async_trait::async_trait; +use http::uri::InvalidUri; +use http::StatusCode; +use hyper::{Body, Method, Request}; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; +use tracing::{debug, error}; +use uuid::Uuid; + +use super::Target; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AggregationResponse { + pub code: i32, + pub msg: Option, + data: Option>, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AggregationRecord { + pub account_id: String, // Firefly Account ID + pub uid: Option, // ID of the identity + pub platform: String, // Platform in [ethereum, farcaster, twitter] + pub identity: String, + pub data_source: String, // firefly or admin(manually_added) + pub update_time: i64, // record update time + pub display_name: String, // fname +} + +pub struct Firefly {} + +#[async_trait] +impl Fetcher for Firefly { + async fn fetch(target: &Target) -> Result { + if !Self::can_fetch(target) { + return Ok(vec![]); + } + Ok(vec![]) + } + + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + match target { + Target::Identity(platform, identity) => { + batch_fetch_connections(platform, identity).await + } + Target::NFT(_, _, _, _) => todo!(), + } + } + + fn can_fetch(target: &Target) -> bool { + target.in_platform_supported(vec![ + Platform::Ethereum, + Platform::Twitter, + Platform::Farcaster, + ]) + } +} + +async fn batch_fetch_connections( + platform: &Platform, + identity: &str, +) -> Result<(TargetProcessedList, EdgeList), Error> { + let records = search_records(platform, identity).await?; + if records.is_empty() { + debug!("Aggregation search result is empty"); + return Ok((vec![], vec![])); + } + debug!("Aggregation search records found {}.", records.len(),); + let mut next_targets: Vec = Vec::new(); + let mut edges: Vec = Vec::new(); + let hv = IdentitiesGraph::default(); + + for (from_idx, from_v) in records.iter().enumerate() { + let mut data_source = DataSource::Firefly; + if from_v.data_source == String::from("admin") { + data_source = DataSource::ManuallyAdded; + } + let from_update_naive = timestamp_to_naive(from_v.update_time, 0); + let from_platform = + Platform::from_str(from_v.platform.as_str()).unwrap_or(Platform::Unknown); + if from_platform == Platform::Unknown { + continue; + } + if from_platform != *platform { + // Do not push duplicate targets into fetchjob + next_targets.push(Target::Identity(from_platform, from_v.identity.clone())) + } + let from = Identity { + uuid: Some(Uuid::new_v4()), + platform: from_platform.clone(), + identity: from_v.identity.clone(), + uid: from_v.uid.clone(), + created_at: from_update_naive, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + for (to_idx, to_v) in records.iter().enumerate() { + if to_idx >= from_idx { + continue; + } + let to_update_naive = timestamp_to_naive(to_v.update_time, 0); + let to_platform = + Platform::from_str(to_v.platform.as_str()).unwrap_or(Platform::Unknown); + if to_platform == Platform::Unknown { + continue; + } + let to = Identity { + uuid: Some(Uuid::new_v4()), + platform: to_platform.clone(), + identity: to_v.identity.clone(), + uid: to_v.uid.clone(), + created_at: to_update_naive.clone(), + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + // add identity connected to hyper vertex + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &from, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &to, HYPER_EDGE), + )); + + let proof_forward = Proof { + uuid: Uuid::new_v4(), + source: data_source, + level: ProofLevel::VeryConfident, + record_id: Some(from_v.account_id.clone()), + created_at: to_update_naive.clone(), + updated_at: naive_now(), + fetcher: DataFetcher::DataMgrService, + }; + + let proof_backward = Proof { + uuid: Uuid::new_v4(), + source: data_source, + level: ProofLevel::VeryConfident, + record_id: Some(from_v.account_id.clone()), + created_at: to_update_naive.clone(), + updated_at: naive_now(), + fetcher: DataFetcher::DataMgrService, + }; + + let pf = proof_forward.wrapper(&from, &to, PROOF_EDGE); + let pb = proof_backward.wrapper(&to, &from, PROOF_REVERSE_EDGE); + + edges.push(EdgeWrapperEnum::new_proof_forward(pf)); + edges.push(EdgeWrapperEnum::new_proof_backward(pb)); + } + } + + Ok((next_targets, edges)) +} + +async fn search_records( + platform: &Platform, + identity: &str, +) -> Result, Error> { + let client = make_client(); + let uri: http::Uri = format!( + "{}/aggregation/search?platform={}&identity={}", + C.upstream.aggregation_service.url.clone(), + platform.to_string(), + identity + ) + .parse() + .map_err(|_err: InvalidUri| Error::ParamError(format!("Uri format Error {}", _err)))?; + + let req = Request::builder() + .method(Method::GET) + .uri(uri) + .body(Body::empty()) + .map_err(|_err| { + Error::ParamError(format!("Aggregation search Build Request Error {}", _err)) + })?; + + let mut resp = request_with_timeout(&client, req, None) + .await + .map_err(|err| { + Error::ManualHttpClientError(format!( + "Aggregation search | error: {:?}", + err.to_string() + )) + })?; + + if !resp.status().is_success() { + let err_message = format!("Aggregation search error, statusCode: {}", resp.status()); + error!(err_message); + return Err(Error::General(err_message, resp.status())); + } + + let result = match parse_body::(&mut resp).await { + Ok(result) => { + if result.code != 0 { + let err_message = format!( + "Aggregation search error | Code: {:?}, Message: {:?}", + result.code, result.msg + ); + error!(err_message); + return Err(Error::General( + err_message, + StatusCode::INTERNAL_SERVER_ERROR, + )); + } + let return_data: Vec = result.data.map_or(vec![], |res| res); + return_data + } + Err(err) => { + let err_message = format!("Genome get_address error parse_body error: {:?}", err); + error!(err_message); + return Err(Error::General(err_message, resp.status())); + } + }; + + Ok(result) +} diff --git a/src/upstream/firefly/tests.rs b/src/upstream/firefly/tests.rs new file mode 100644 index 00000000..987c9e2f --- /dev/null +++ b/src/upstream/firefly/tests.rs @@ -0,0 +1,33 @@ +#[cfg(test)] +mod tests { + use crate::error::Error; + use crate::upstream::firefly::{search_records, Firefly}; + use crate::upstream::{Fetcher, Platform, Target}; + + #[tokio::test] + async fn test_search_records() -> Result<(), Error> { + let _identity = "kins"; + let _platform = Platform::Farcaster; + + let _identity_1 = "j0hnwang"; + let _platform_1 = Platform::Twitter; + + let _identity_2 = "0x88a4febb4572cf01967e5ff9b6109dea57168c6d"; + let _platform_2 = Platform::Ethereum; + + let records = search_records(&_platform_1, _identity_1).await?; + println!("records: {:?}", records); + Ok(()) + } + + #[tokio::test] + async fn test_fetch() -> Result<(), Error> { + let target = Target::Identity( + Platform::Ethereum, + "0x61ae970ac67ff4164ebf2fd6f38f630df522e5ef".to_lowercase(), + ); + // let target = Target::Identity(Platform::Farcaster, "kins".to_string()); + let _ = Firefly::fetch(&target).await?; + Ok(()) + } +} diff --git a/src/upstream/genome/mod.rs b/src/upstream/genome/mod.rs index 6173e6ec..732b6266 100644 --- a/src/upstream/genome/mod.rs +++ b/src/upstream/genome/mod.rs @@ -2,13 +2,16 @@ mod tests; use crate::config::C; use crate::error::Error; -use crate::tigergraph::edge::{Hold, Resolve}; +use crate::tigergraph::edge::{ + Hold, HyperEdge, Resolve, Wrapper, HOLD_CONTRACT, HOLD_IDENTITY, HYPER_EDGE, RESOLVE, + REVERSE_RESOLVE, +}; use crate::tigergraph::upsert::create_ens_identity_ownership; use crate::tigergraph::upsert::create_identity_domain_resolve_record; use crate::tigergraph::upsert::create_identity_domain_reverse_resolve_record; use crate::tigergraph::upsert::create_identity_to_contract_hold_record; -// use crate::tigergraph::upsert::create_identity_to_identity_hold_record; -use crate::tigergraph::vertex::{Contract, Identity}; +use crate::tigergraph::vertex::{Contract, IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; use crate::upstream::{ Chain, ContractCategory, DataFetcher, DataSource, DomainNameSystem, Fetcher, Platform, Target, TargetProcessedList, @@ -21,7 +24,7 @@ use http::uri::InvalidUri; use http::StatusCode; use hyper::{Body, Method, Request}; use serde::{Deserialize, Serialize}; -use tracing::{debug, error}; +use tracing::{debug, error, info}; use uuid::Uuid; #[derive(Debug, Clone, Deserialize, Serialize)] @@ -72,11 +75,258 @@ impl Fetcher for Genome { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + match target.platform()? { + Platform::Ethereum => batch_fetch_by_address(target).await, + Platform::Genome => batch_fetch_by_domain(target).await, + _ => Ok((vec![], vec![])), + } + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::Genome, Platform::Ethereum]) } } +async fn batch_fetch_by_address(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let address = target.identity()?.to_lowercase(); + let domains = get_name(&address).await?; + if domains.is_empty() { + debug!(?target, "Genome get_name result is empty"); + return Ok((vec![], vec![])); + } + + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + for d in domains.into_iter() { + let genome_domain = format!("{}.{}", d.name, d.tld_name); + let mut token_id = String::from(""); + let expired_at_naive = timestamp_to_naive(d.expired_at, 0); + + if let Some(_token_id) = d.token_id.clone() { + token_id = _token_id; + } + + let gno: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Genome, + identity: genome_domain.clone(), + uid: None, + created_at: None, + display_name: Some(genome_domain.clone()), + added_at: naive_now(), + avatar_url: d.image_url.clone(), + profile_url: None, + updated_at: naive_now(), + expired_at: expired_at_naive, + reverse: Some(d.is_default.clone()), + }; + + let addr: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Ethereum, + identity: d.owner.clone().to_lowercase(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(d.is_default.clone()), + }; + + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::SpaceId, + transaction: Some("".to_string()), + id: token_id, + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::DataMgrService, + expired_at: expired_at_naive, + }; + + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::SpaceId, + system: DomainNameSystem::Genome, + name: genome_domain.clone(), + fetcher: DataFetcher::DataMgrService, + updated_at: naive_now(), + }; + + let contract = Contract { + uuid: Uuid::new_v4(), + category: ContractCategory::GNS, + address: ContractCategory::GNS.default_contract_address().unwrap(), + chain: Chain::Gnosis, + symbol: Some("GNS".to_string()), + updated_at: naive_now(), + }; + + if d.is_default { + // 'reverse' resolution maps from an address back to a name. + let reverse: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::SpaceId, + system: DomainNameSystem::Genome, + name: genome_domain.clone(), + fetcher: DataFetcher::DataMgrService, + updated_at: naive_now(), + }; + debug!("{} => Genome({}) is_default", address, genome_domain); + let rr = reverse.wrapper(&addr, &gno, REVERSE_RESOLVE); + edges.push(EdgeWrapperEnum::new_reverse_resolve(rr)); + } + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &addr, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &gno, HYPER_EDGE), + )); + + // 'regular' resolution involves mapping from a name to an address. + let rs = resolve.wrapper(&gno, &addr, RESOLVE); + // hold record + let hd = hold.wrapper(&addr, &gno, HOLD_IDENTITY); + let hdc = hold.wrapper(&addr, &contract, HOLD_CONTRACT); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_hold_contract(hdc)); + } + + // after genome, nothing return for next target + Ok((vec![], edges)) +} + +async fn batch_fetch_by_domain(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let name = target.identity()?.to_lowercase(); + let name_with_out_tld: &str = name.trim_end_matches(".gno"); + let domains: Vec = get_address(name_with_out_tld).await?; // get_address(domain) + if domains.is_empty() { + debug!(?target, "Genome get_address result is empty"); + return Ok((vec![], vec![])); + } + let address = domains.first().unwrap().owner.clone(); + let mut next_targets = TargetProcessedList::new(); + next_targets.push(Target::Identity( + Platform::Ethereum, + address.clone().to_lowercase(), + )); + + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + for d in domains.into_iter() { + let genome_domain = format!("{}.{}", d.name, d.tld_name); + let mut token_id = String::from(""); + let expired_at_naive = timestamp_to_naive(d.expired_at, 0); + + if let Some(_token_id) = d.token_id.clone() { + token_id = _token_id; + } + let gno: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Genome, + identity: genome_domain.clone(), + uid: None, + created_at: None, + display_name: Some(genome_domain.clone()), + added_at: naive_now(), + avatar_url: d.image_url.clone(), + profile_url: None, + updated_at: naive_now(), + expired_at: expired_at_naive, + reverse: Some(d.is_default.clone()), + }; + + let addr: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Ethereum, + identity: d.owner.clone().to_lowercase(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(d.is_default.clone()), + }; + + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::SpaceId, + transaction: Some("".to_string()), + id: token_id, + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::DataMgrService, + expired_at: expired_at_naive, + }; + + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::SpaceId, + system: DomainNameSystem::Genome, + name: genome_domain.clone(), + fetcher: DataFetcher::DataMgrService, + updated_at: naive_now(), + }; + + let contract = Contract { + uuid: Uuid::new_v4(), + category: ContractCategory::GNS, + address: ContractCategory::GNS.default_contract_address().unwrap(), + chain: Chain::Gnosis, + symbol: Some("GNS".to_string()), + updated_at: naive_now(), + }; + + if d.is_default { + // 'reverse' resolution maps from an address back to a name. + let reverse: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::SpaceId, + system: DomainNameSystem::Genome, + name: genome_domain.clone(), + fetcher: DataFetcher::DataMgrService, + updated_at: naive_now(), + }; + debug!("{} => Genome({}) is_default", address, genome_domain); + let rr = reverse.wrapper(&addr, &gno, REVERSE_RESOLVE); + edges.push(EdgeWrapperEnum::new_reverse_resolve(rr)); + } + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &addr, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &gno, HYPER_EDGE), + )); + + // 'regular' resolution involves mapping from a name to an address. + let rs = resolve.wrapper(&gno, &addr, RESOLVE); + // hold record + let hd = hold.wrapper(&addr, &gno, HOLD_IDENTITY); + let hdc = hold.wrapper(&addr, &contract, HOLD_CONTRACT); + info!("hdc = {:?}", hdc); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_hold_contract(hdc)); + } + + Ok((next_targets, edges)) +} + async fn fetch_connections_by_platform_identity( platform: &Platform, identity: &str, @@ -147,7 +397,7 @@ async fn fetch_domain_by_address( let hold: Hold = Hold { uuid: Uuid::new_v4(), source: DataSource::SpaceId, - transaction: None, + transaction: Some("".to_string()), id: token_id, created_at: None, updated_at: naive_now(), @@ -259,7 +509,7 @@ async fn fetch_address_by_domain( let hold: Hold = Hold { uuid: Uuid::new_v4(), source: DataSource::SpaceId, - transaction: None, + transaction: Some("".to_string()), id: token_id, created_at: None, updated_at: naive_now(), diff --git a/src/upstream/keybase/mod.rs b/src/upstream/keybase/mod.rs index 7de8ee40..8f52ed09 100644 --- a/src/upstream/keybase/mod.rs +++ b/src/upstream/keybase/mod.rs @@ -3,10 +3,13 @@ mod tests; use crate::config::C; use crate::error::Error; -use crate::tigergraph::edge::Proof; +use crate::tigergraph::edge::{ + HyperEdge, Proof, Wrapper, HYPER_EDGE, PROOF_EDGE, PROOF_REVERSE_EDGE, +}; use crate::tigergraph::upsert::create_identity_to_identity_proof_two_way_binding; -use crate::tigergraph::vertex::{Identity, IdentityRecord}; +use crate::tigergraph::vertex::{IdentitiesGraph, Identity, IdentityRecord}; use crate::tigergraph::{BaseResponse, Graph}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; use crate::upstream::{DataSource, Fetcher, Platform, ProofLevel, TargetProcessedList}; use crate::util::{ make_client, make_http_client, naive_now, option_naive_datetime_from_string, @@ -132,6 +135,18 @@ impl Fetcher for Keybase { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + match target { + Target::Identity(platform, identity) => { + stable_batch_fetch_connections(platform, identity).await + } + Target::NFT(_, _, _, _) => Ok((vec![], vec![])), + } + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![ Platform::Twitter, @@ -225,6 +240,147 @@ async fn fake_fetch_connections_by_platform_identity( Ok(next_targets) } +async fn stable_batch_fetch_connections( + platform: &Platform, + identity: &str, +) -> Result<(TargetProcessedList, EdgeList), Error> { + // BTC Character case sensitive + let mut format_identity = identity.to_string(); + if platform.to_owned() != Platform::Bitcoin { + format_identity = format_identity.to_lowercase(); + } + + let mut next_targets: TargetProcessedList = Vec::new(); + let mut edges: Vec = Vec::new(); + let hv = IdentitiesGraph::default(); + + let uri: http::Uri = format!( + "{}/proofs_summary?platform={}&username={}", + C.upstream.keybase_service.stable_url, platform, format_identity + ) + .parse() + .map_err(|_err: InvalidUri| { + Error::ParamError(format!( + "{}={} Uri format Error | {}", + platform, format_identity, _err + )) + })?; + let req = hyper::Request::builder() + .method(Method::GET) + .uri(uri) + .body(Body::empty()) + .map_err(|_err| Error::ParamError(format!("ParamError Error | {}", _err)))?; + + let client = make_http_client(); + let mut resp = client.request(req).await.map_err(|err| { + Error::ManualHttpClientError(format!( + "Keybase proofs_summary?platform={}&identity={} error | Fail to request: {:?}", + platform, + format_identity, + err.to_string() + )) + })?; + + let proofs = match parse_body::(&mut resp).await { + Ok(r) => { + if r.code != 0 { + let err_message = format!( + "Keybase proofs_summary error | Code: {:?}, Message: {:?}", + r.code, r.msg + ); + error!(err_message); + return Err(Error::General( + err_message, + StatusCode::INTERNAL_SERVER_ERROR, + )); + } + let result = r.data.map_or(vec![], |res| res); + // tracing::debug!("proofs_summary result {:?}", result); + debug!("Keybase proofs_summary = {} Records found.", result.len(),); + result + } + Err(err) => { + let err_message = format!("Keybase proofs_summary error parse_body error: {:?}", err); + error!(err_message); + return Err(Error::General(err_message, resp.status())); + } + }; + + for p in proofs.into_iter() { + let to_platform = Platform::from_str(&p.platform.as_str()).unwrap_or(Platform::Unknown); + if to_platform == Platform::Unknown { + continue; + } + if to_platform != platform.to_owned() && to_platform == Platform::Twitter { + next_targets.push(Target::Identity(to_platform, p.username.clone())); + } + + let from: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Keybase, + identity: p.keybase_username.clone(), + uid: None, + created_at: None, + display_name: Some(p.keybase_username.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let to: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: to_platform.clone(), + identity: p.username.clone(), + uid: None, + created_at: None, + display_name: p.display_name, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let proof_forward: Proof = Proof { + uuid: Uuid::new_v4(), + source: DataSource::Keybase, + level: ProofLevel::VeryConfident, + record_id: p.record_id.clone(), + created_at: p.created_time.clone(), + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + }; + + let proof_backward: Proof = Proof { + uuid: Uuid::new_v4(), + source: DataSource::Keybase, + level: ProofLevel::VeryConfident, + record_id: p.record_id.clone(), + created_at: p.created_time.clone(), + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + }; + + // add identity connected to hyper vertex + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &from, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &to, HYPER_EDGE), + )); + let pf = proof_forward.wrapper(&from, &to, PROOF_EDGE); + let pb = proof_backward.wrapper(&to, &from, PROOF_REVERSE_EDGE); + edges.push(EdgeWrapperEnum::new_proof_forward(pf)); + edges.push(EdgeWrapperEnum::new_proof_backward(pb)); + } + + Ok((next_targets, edges)) +} + async fn stable_fetch_connections_by_platform_identity( platform: &Platform, identity: &str, diff --git a/src/upstream/knn3/mod.rs b/src/upstream/knn3/mod.rs index 55878fa7..13b70bf1 100644 --- a/src/upstream/knn3/mod.rs +++ b/src/upstream/knn3/mod.rs @@ -6,6 +6,7 @@ use crate::error::Error; use crate::tigergraph::edge::Hold; use crate::tigergraph::upsert::create_identity_to_contract_hold_record; use crate::tigergraph::vertex::{Contract, Identity}; +use crate::tigergraph::EdgeList; use crate::upstream::{ Chain, ContractCategory, DataFetcher, DataSource, Fetcher, Platform, Target, TargetProcessedList, @@ -63,6 +64,13 @@ impl Fetcher for Knn3 { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + Ok((vec![], vec![])) + } + fn can_fetch(_target: &Target) -> bool { // TODO: temporarily disable KNN3 fetcher false diff --git a/src/upstream/lens/mod.rs b/src/upstream/lens/mod.rs index 9e3daba9..dd7272f2 100644 --- a/src/upstream/lens/mod.rs +++ b/src/upstream/lens/mod.rs @@ -8,6 +8,7 @@ use crate::tigergraph::upsert::create_identity_domain_resolve_record; use crate::tigergraph::upsert::create_identity_domain_reverse_resolve_record; use crate::tigergraph::upsert::create_identity_to_identity_hold_record; use crate::tigergraph::vertex::Identity; +use crate::tigergraph::EdgeList; use crate::upstream::{ DataFetcher, DataSource, DomainNameSystem, Fetcher, Platform, Target, TargetProcessedList, }; @@ -116,6 +117,13 @@ impl Fetcher for Lens { } } + async fn batch_fetch(_target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + Ok((vec![], vec![])) + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::Ethereum, Platform::Lens]) } diff --git a/src/upstream/lensv2/mod.rs b/src/upstream/lensv2/mod.rs index 0e295778..8984c711 100644 --- a/src/upstream/lensv2/mod.rs +++ b/src/upstream/lensv2/mod.rs @@ -3,11 +3,14 @@ mod tests; use crate::config::C; use crate::error::Error; -use crate::tigergraph::edge::{Hold, Resolve}; +use crate::tigergraph::edge::{ + Hold, HyperEdge, Resolve, Wrapper, HOLD_IDENTITY, HYPER_EDGE, RESOLVE, REVERSE_RESOLVE, +}; use crate::tigergraph::upsert::create_identity_domain_resolve_record; use crate::tigergraph::upsert::create_identity_domain_reverse_resolve_record; use crate::tigergraph::upsert::create_identity_to_identity_hold_record; -use crate::tigergraph::vertex::Identity; +use crate::tigergraph::vertex::{IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; use crate::upstream::{ DataFetcher, DataSource, DomainNameSystem, Fetcher, Platform, Target, TargetProcessedList, }; @@ -142,11 +145,315 @@ impl Fetcher for LensV2 { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + match target.platform()? { + Platform::Ethereum => batch_fetch_by_wallet(target).await, + Platform::Lens => batch_fetch_by_handle(target).await, + _ => Ok((vec![], vec![])), + } + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::Ethereum, Platform::Lens]) } } +async fn query_by_handle(handle_name: &str) -> Result, Error> { + let operation = ProfileQueryByHandles::build(ProfilesRequestVariables { + handles: Some(vec![Handle(handle_name.to_string())]), + owned_by: None, + }); + let response = surf::post(C.upstream.lens_api.url.clone()) + .run_graphql(operation) + .await; + if response.is_err() { + warn!( + "LensV2 {} | Failed to fetch: {}", + handle_name, + response.unwrap_err(), + ); + return Ok(vec![]); + } + + let profiles = response + .unwrap() + .data + .map_or(vec![], |data| data.profiles.items); + + Ok(profiles) +} + +async fn query_by_wallet(wallet: &str) -> Result, Error> { + let operation = ProfileQueryByHandles::build(ProfilesRequestVariables { + handles: None, + owned_by: Some(vec![EvmAddress(wallet.to_string())]), + }); + let response = surf::post(C.upstream.lens_api.url.clone()) + .run_graphql(operation) + .await; + + if response.is_err() { + warn!( + "LensV2 {} | Failed to fetch: {}", + wallet, + response.unwrap_err(), + ); + return Ok(vec![]); + } + let profiles = response + .unwrap() + .data + .map_or(vec![], |data| data.profiles.items); + + Ok(profiles) +} + +async fn batch_fetch_by_handle(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let target_var = target.identity()?; + let handle_name = target_var.trim_end_matches(".lens"); + let full_handle = format!("lens/{}", handle_name); + let profiles = query_by_handle(&full_handle).await?; + if profiles.is_empty() { + warn!("LensV2 target {} | No Result", target,); + return Ok((vec![], vec![])); + } + let lens_profile = profiles.first().unwrap().clone(); + if lens_profile.handle.clone().is_none() { + warn!("LensV2 target {} | lens handle is null", target,); + return Ok((vec![], vec![])); + } + + let mut next_targets = TargetProcessedList::new(); + let hv = IdentitiesGraph::default(); + let mut edges = EdgeList::new(); + + let evm_owner = lens_profile.owned_by.address.0.to_ascii_lowercase(); + // fetch default profile id for lens-v2 + let default_profile_id = get_default_profile_id(&evm_owner).await?; + + let mut is_default = false; + if let Some(default_profile_id) = default_profile_id { + if lens_profile.id == default_profile_id { + trace!( + "LensV2 target {} | profile.id {:?} == default_profile_id {:?}", + target, + lens_profile.id, + default_profile_id + ); + is_default = true; + } + } + let handle_info = lens_profile.handle.clone().unwrap(); + let lens_handle = format!("{}.{}", handle_info.local_name, handle_info.namespace); + let lens_display_name = lens_profile + .metadata + .clone() + .map_or(None, |metadata| metadata.display_name); + let created_at = utc_to_naive(lens_profile.created_at.clone().0)?; + + let addr: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Ethereum, + identity: evm_owner.clone(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(is_default), + }; + + let lens: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Lens, + identity: lens_handle.clone(), + uid: Some(lens_profile.id.clone().0.to_string()), + created_at: Some(created_at), + display_name: lens_display_name, + added_at: naive_now(), + avatar_url: None, + profile_url: Some("https://hey.xyz/u/".to_owned() + &handle_info.local_name), + updated_at: naive_now(), + expired_at: None, + reverse: Some(is_default), + }; + + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::Lens, + transaction: Some(lens_profile.tx_hash.clone().0), + id: lens_profile.id.clone().0.to_string(), + created_at: Some(created_at), + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: None, + }; + + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::Lens, + system: DomainNameSystem::Lens, + name: lens_handle.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + + if is_default { + // field `is_default` has been canceled in lens-v2-api + // It is an independent query `GetDefaultProfile` and is not returned in the profile field. + let reverse: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::Lens, + system: DomainNameSystem::Lens, + name: lens_handle.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + let rrs = reverse.wrapper(&addr, &lens, REVERSE_RESOLVE); + edges.push(EdgeWrapperEnum::new_reverse_resolve(rrs)); + } + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &addr, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &lens, HYPER_EDGE), + )); + + let hd = hold.wrapper(&addr, &lens, HOLD_IDENTITY); + let rs = resolve.wrapper(&lens, &addr, RESOLVE); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + + next_targets.push(Target::Identity(Platform::Ethereum, evm_owner.clone())); + + Ok((next_targets, edges)) +} + +async fn batch_fetch_by_wallet(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let target_var = target.identity()?; + let owned_by_evm = target_var.to_lowercase(); + let profiles = query_by_wallet(&owned_by_evm).await?; + if profiles.is_empty() { + warn!("LensV2 target {} | No Result", target,); + return Ok((vec![], vec![])); + } + + // fetch default profile id for lens-v2 + let default_profile_id = get_default_profile_id(&owned_by_evm).await?; + let hv = IdentitiesGraph::default(); + let mut edges = EdgeList::new(); + + for lens_profile in profiles.iter() { + let mut is_default = false; + if let Some(default_profile_id) = default_profile_id.clone() { + if lens_profile.id == default_profile_id { + trace!( + "LensV2 target {} | profile.id {:?} == default_profile_id {:?}", + target, + lens_profile.id, + default_profile_id + ); + is_default = true; + } + } + let evm_owner = lens_profile.owned_by.address.0.to_ascii_lowercase(); + let handle_info = lens_profile.handle.clone().unwrap(); + let lens_handle = format!("{}.{}", handle_info.local_name, handle_info.namespace); + let lens_display_name = lens_profile + .metadata + .clone() + .map_or(None, |metadata| metadata.display_name); + let created_at = utc_to_naive(lens_profile.created_at.clone().0)?; + + let addr: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Ethereum, + identity: evm_owner.clone(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(is_default), + }; + + let lens: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Lens, + identity: lens_handle.clone(), + uid: Some(lens_profile.id.clone().0.to_string()), + created_at: Some(created_at), + display_name: lens_display_name, + added_at: naive_now(), + avatar_url: None, + profile_url: Some("https://hey.xyz/u/".to_owned() + &handle_info.local_name), + updated_at: naive_now(), + expired_at: None, + reverse: Some(is_default), + }; + + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::Lens, + transaction: Some(lens_profile.tx_hash.clone().0), + id: lens_profile.id.clone().0.to_string(), + created_at: Some(created_at), + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: None, + }; + + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::Lens, + system: DomainNameSystem::Lens, + name: lens_handle.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + + if is_default { + // field `is_default` has been canceled in lens-v2-api + // It is an independent query `GetDefaultProfile` and is not returned in the profile field. + let reverse: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::Lens, + system: DomainNameSystem::Lens, + name: lens_handle.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + let rrs = reverse.wrapper(&addr, &lens, REVERSE_RESOLVE); + edges.push(EdgeWrapperEnum::new_reverse_resolve(rrs)); + } + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &addr, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &lens, HYPER_EDGE), + )); + + let hd = hold.wrapper(&addr, &lens, HOLD_IDENTITY); + let rs = resolve.wrapper(&lens, &addr, RESOLVE); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + } + + Ok((vec![], edges)) +} + async fn fetch_by_lens_handle(target: &Target) -> Result { let target_var = target.identity()?; let handle_name = target_var.trim_end_matches(".lens"); diff --git a/src/upstream/lensv2/tests.rs b/src/upstream/lensv2/tests.rs index 3dd827ce..4f3a5527 100644 --- a/src/upstream/lensv2/tests.rs +++ b/src/upstream/lensv2/tests.rs @@ -26,13 +26,13 @@ mod tests { #[tokio::test] async fn test_fetch_by_lens_handle() -> Result<(), Error> { - let target = Target::Identity(Platform::Lens, String::from("sujiyan.lens")); + let target = Target::Identity(Platform::Lens, String::from("xnownx.lens")); let _ = LensV2::fetch(&target).await?; let client = make_http_client(); let found = Identity::find_by_platform_identity( &client, &Platform::Ethereum, - &String::from("0x934B510D4C9103E6a87AEf13b816fb080286D649").to_lowercase(), + &String::from("0x88a4FebB4572CF01967e5Ff9B6109dEA57168c6d").to_lowercase(), ) .await? .expect("Record not found"); diff --git a/src/upstream/mod.rs b/src/upstream/mod.rs index 1cee3256..ac68b7fc 100644 --- a/src/upstream/mod.rs +++ b/src/upstream/mod.rs @@ -14,6 +14,8 @@ mod solana; mod space_id; mod sybil_list; mod unstoppable; +// mod firefly; +// mod opensea; #[cfg(test)] mod tests; @@ -22,14 +24,14 @@ mod types; use crate::{ error::Error, + tigergraph::{batch_upsert, EdgeList}, upstream::{ - aggregation::Aggregation, crossbell::Crossbell, dotbit::DotBit, - ens_reverse::ENSReverseLookup, farcaster::Farcaster, genome::Genome, keybase::Keybase, - knn3::Knn3, lensv2::LensV2, proof_client::ProofClient, rss3::Rss3, solana::Solana, - space_id::SpaceId, sybil_list::SybilList, the_graph::TheGraph, + crossbell::Crossbell, dotbit::DotBit, ens_reverse::ENSReverseLookup, farcaster::Farcaster, + genome::Genome, keybase::Keybase, knn3::Knn3, lensv2::LensV2, proof_client::ProofClient, + rss3::Rss3, solana::Solana, space_id::SpaceId, sybil_list::SybilList, the_graph::TheGraph, unstoppable::UnstoppableDomains, }, - util::hashset_append, + util::{hashset_append, make_http_client}, }; use async_trait::async_trait; use futures::{future::join_all, StreamExt}; @@ -54,6 +56,9 @@ pub trait Fetcher { /// Fetch data from given source. async fn fetch(target: &Target) -> Result; + /// Fetch all vertices and edges from given source and return them + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error>; + /// Determine if this upstream can fetch this target. fn can_fetch(target: &Target) -> bool; } @@ -66,9 +71,11 @@ pub trait Fetcher { #[async_recursion::async_recursion] pub async fn fetch_all(targets: TargetProcessedList, depth: Option) -> Result<(), Error> { let mut round: u16 = 0; + let mut up_next: HashSet = HashSet::new(); + let mut all_edges: EdgeList = EdgeList::new(); let mut fetching = FETCHING.lock().await; - let mut up_next: HashSet = HashSet::from_iter( + up_next.extend( targets .clone() .into_iter() @@ -76,19 +83,14 @@ pub async fn fetch_all(targets: TargetProcessedList, depth: Option) -> Resu ); up_next.iter().for_each(|target| { fetching.insert(target.clone()); - () }); drop(fetching); - let mut processed: HashSet = HashSet::new(); - // FETCHING.lock().await.insert(initial_target.clone()); - // queues of this session. - // let mut up_next = HashSet::from([initial_target.clone()]); - // let mut processed: HashSet = HashSet::new(); + let mut processed: HashSet = HashSet::new(); - while up_next.len() > 0 { + while !up_next.is_empty() { round += 1; - let result = fetch_many( + let (next_targets, edges) = fetch_many( up_next .clone() .into_iter() @@ -97,24 +99,31 @@ pub async fn fetch_all(targets: TargetProcessedList, depth: Option) -> Resu Some(round), ) .await?; + + hashset_append(&mut processed, up_next.into_iter().collect()); + up_next = HashSet::from_iter(next_targets.into_iter()); + + all_edges.extend(edges); + if depth.is_some() && depth.unwrap() <= round { // Fork as background job to continue fetching. tokio::spawn(fetch_all(up_next.into_iter().collect(), None)); break; } - - // Add previous up_next into processed, and replace with new up_next - hashset_append(&mut processed, up_next.into_iter().collect()); - up_next = HashSet::from_iter(result.into_iter()); } let mut fetching = FETCHING.lock().await; targets.iter().for_each(|target| { fetching.remove(&target); - () }); drop(fetching); + // Upsert all edges after fetching completes + if !all_edges.is_empty() { + let cli = make_http_client(); + batch_upsert(&cli, all_edges).await?; + } + event!( Level::INFO, round, @@ -122,60 +131,155 @@ pub async fn fetch_all(targets: TargetProcessedList, depth: Option) -> Resu processed = processed.len(), "Fetch completed." ); + Ok(()) + + // let mut round: u16 = 0; + // let mut fetching = FETCHING.lock().await; + // let mut up_next: HashSet = HashSet::from_iter( + // targets + // .clone() + // .into_iter() + // .filter(|target| !fetching.contains(target)), + // ); + // up_next.iter().for_each(|target| { + // fetching.insert(target.clone()); + // () + // }); + // drop(fetching); + // let mut processed: HashSet = HashSet::new(); + + // // FETCHING.lock().await.insert(initial_target.clone()); + // // queues of this session. + // // let mut up_next = HashSet::from([initial_target.clone()]); + // // let mut processed: HashSet = HashSet::new(); + + // while up_next.len() > 0 { + // round += 1; + // let result = fetch_many( + // up_next + // .clone() + // .into_iter() + // .filter(|target| !processed.contains(target)) + // .collect(), + // Some(round), + // ) + // .await?; + // if depth.is_some() && depth.unwrap() <= round { + // // Fork as background job to continue fetching. + // tokio::spawn(fetch_all(up_next.into_iter().collect(), None)); + // break; + // } + + // // Add previous up_next into processed, and replace with new up_next + // hashset_append(&mut processed, up_next.into_iter().collect()); + // up_next = HashSet::from_iter(result.into_iter()); + // } + // let mut fetching = FETCHING.lock().await; + // targets.iter().for_each(|target| { + // fetching.remove(&target); + // () + // }); + // drop(fetching); + // event!( + // Level::INFO, + // round, + // ?depth, + // processed = processed.len(), + // "Fetch completed." + // ); + // Ok(()) } /// Fetch targets in parallel of 5. /// `round` is only for log purpose. -pub async fn fetch_many(targets: Vec, round: Option) -> Result, Error> { +pub async fn fetch_many( + targets: Vec, + round: Option, +) -> Result<(TargetProcessedList, EdgeList), Error> { const CONCURRENT: usize = 5; - let futures: Vec<_> = targets.iter().map(|target| fetch_one(target)).collect(); + let futures: Vec<_> = targets + .iter() + .map(|target| batch_fetch_upstream(target)) + .collect(); let futures_stream = futures::stream::iter(futures).buffer_unordered(CONCURRENT); - - let mut result: TargetProcessedList = futures_stream - .collect::, Error>>>() - .await - .into_iter() - .flat_map(|handle_result| -> Vec { - match handle_result { - Ok(result) => { - event!( - Level::DEBUG, - ?round, - fetched_length = result.len(), - "Round completed." - ); - result - } - Err(err) => { - event!(Level::WARN, ?round, %err, "Error happened in fetching task"); - vec![] + let (mut all_targets, all_edges) = futures_stream + .fold( + (TargetProcessedList::new(), EdgeList::new()), + |(mut all_targets, mut all_edges), handle_result| async move { + match handle_result { + Ok((targets, edges)) => { + event!( + Level::DEBUG, + ?round, + fetched_length = targets.len(), + "Round completed." + ); + all_targets.extend(targets); + all_edges.extend(edges); + } + Err(err) => { + event!(Level::WARN, ?round, %err, "Error happened in fetching task"); + } } - } - }) - .collect(); - result.dedup(); - Ok(result) + (all_targets, all_edges) + }, + ) + .await; + all_targets.dedup(); + + // Instead of upsert edges after each `Round completed`, + // wait for all data sources to be added after fetch_all ends. + Ok((all_targets, all_edges)) + + // const CONCURRENT: usize = 5; + // let futures: Vec<_> = targets.iter().map(|target| fetch_one(target)).collect(); + // let futures_stream = futures::stream::iter(futures).buffer_unordered(CONCURRENT); + + // let mut result: TargetProcessedList = futures_stream + // .collect::, Error>>>() + // .await + // .into_iter() + // .flat_map(|handle_result| -> Vec { + // match handle_result { + // Ok(result) => { + // event!( + // Level::DEBUG, + // ?round, + // fetched_length = result.len(), + // "Round completed." + // ); + // result + // } + // Err(err) => { + // event!(Level::WARN, ?round, %err, "Error happened in fetching task"); + // vec![] + // } + // } + // }) + // .collect(); + // result.dedup(); + // Ok(result) } + /// Find one (platform, identity) pair in all upstreams. /// Returns amount of identities just fetched for next iter. pub async fn fetch_one(target: &Target) -> Result, Error> { let mut up_next: TargetProcessedList = join_all(vec![ + TheGraph::fetch(target), + ENSReverseLookup::fetch(target), Farcaster::fetch(target), LensV2::fetch(target), - Aggregation::fetch(target), - SybilList::fetch(target), - Keybase::fetch(target), ProofClient::fetch(target), + Keybase::fetch(target), + SybilList::fetch(target), Rss3::fetch(target), Knn3::fetch(target), - ENSReverseLookup::fetch(target), DotBit::fetch(target), UnstoppableDomains::fetch(target), SpaceId::fetch(target), Genome::fetch(target), Crossbell::fetch(target), - TheGraph::fetch(target), Solana::fetch(target), ]) .await @@ -207,6 +311,64 @@ pub async fn fetch_one(target: &Target) -> Result, Error> { Ok(up_next) } +pub async fn batch_fetch_upstream( + target: &Target, +) -> Result<(TargetProcessedList, EdgeList), Error> { + let mut up_next = TargetProcessedList::new(); + let mut all_edges = EdgeList::new(); + + let _ = join_all(vec![ + TheGraph::batch_fetch(target), + ENSReverseLookup::batch_fetch(target), + Farcaster::batch_fetch(target), + LensV2::batch_fetch(target), + ProofClient::batch_fetch(target), + Keybase::batch_fetch(target), + Rss3::batch_fetch(target), + DotBit::batch_fetch(target), + UnstoppableDomains::batch_fetch(target), + SpaceId::batch_fetch(target), + Genome::batch_fetch(target), + Crossbell::batch_fetch(target), + Solana::batch_fetch(target), + // SybilList::batch_fetch(target), // move this logic to `data_process` as a scheduled asynchronous fetch + // Knn3::batch_fetch(target), // Temporarily cancel + // Firefly::batch_fetch(target), // Temporarily cancel + // OpenSea::batch_fetch(target), // Temporarily cancel + ]) + .await + .into_iter() + .for_each(|res| { + if let Ok((next_targets, edges)) = res { + up_next.extend(next_targets); + all_edges.extend(edges); + } else if let Err(err) = res { + warn!( + "Error happened when fetching and saving {}: {}", + target, err + ); + // Don't break the procedure, continue with other results + } + }); + + up_next.dedup(); + // Filter zero address + up_next = up_next + .into_iter() + .filter(|target| match target { + Target::Identity(Platform::Ethereum, address) => { + // Filter zero address (without last 4 digits) + return !address.starts_with("0x000000000000000000000000000000000000"); + } + Target::Identity(_, _) => true, + Target::NFT(_, _, _, _) => true, + }) + .collect(); + + // event!(Level::INFO, "fetch_one_and_save up_next {:?}", up_next); + Ok((up_next, all_edges)) +} + /// Prefetch all prefetchable upstreams, e.g. SybilList. pub async fn prefetch() -> Result<(), Error> { info!("Prefetching sybil_list ..."); diff --git a/src/upstream/opensea/mod.rs b/src/upstream/opensea/mod.rs new file mode 100644 index 00000000..0e1ce42b --- /dev/null +++ b/src/upstream/opensea/mod.rs @@ -0,0 +1,233 @@ +#[cfg(test)] +mod tests; +use crate::config::C; +use crate::error::Error; +use crate::tigergraph::edge::{HyperEdge, Proof, Wrapper}; +use crate::tigergraph::edge::{HYPER_EDGE, PROOF_EDGE, PROOF_REVERSE_EDGE}; +use crate::tigergraph::vertex::{IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; +use crate::upstream::{ + DataFetcher, DataSource, Fetcher, Platform, ProofLevel, TargetProcessedList, +}; +use crate::util::{make_client, naive_now, parse_body, request_with_timeout}; + +use async_trait::async_trait; +use http::uri::InvalidUri; +use http::StatusCode; +use hyper::{Body, Method, Request}; +use serde::{Deserialize, Serialize}; +use tracing::{debug, error}; +use uuid::Uuid; + +use super::Target; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct DataResponse { + pub code: i32, + pub msg: Option, + pub data: Option>, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct SnsRecord { + pub address: String, // evm address + pub sns_platform: String, // Platform in [twitter, instagram] + pub sns_handle: String, + pub is_verified: bool, // is verified or not +} + +pub struct OpenSea {} + +#[async_trait] +impl Fetcher for OpenSea { + async fn fetch(target: &Target) -> Result { + if !Self::can_fetch(target) { + return Ok(vec![]); + } + Ok(vec![]) + } + + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + match target { + Target::Identity(platform, identity) => { + batch_fetch_connections(platform, identity).await + } + Target::NFT(_, _, _, _) => todo!(), + } + } + + fn can_fetch(target: &Target) -> bool { + target.in_platform_supported(vec![ + Platform::Ethereum, + Platform::Twitter, + Platform::Instagram, + ]) + } +} + +async fn batch_fetch_connections( + platform: &Platform, + identity: &str, +) -> Result<(TargetProcessedList, EdgeList), Error> { + let records = search_opensea_account(platform, identity).await?; + if records.is_empty() { + debug!("OpenSea search result is empty"); + return Ok((vec![], vec![])); + } + debug!("OpenSea search records found {}.", records.len(),); + let mut next_targets: Vec = Vec::new(); + let mut edges: Vec = Vec::new(); + let hv = IdentitiesGraph::default(); + + for record in records.iter() { + let sns_platform: Platform = record.sns_platform.parse()?; + let sns_handle = record.sns_handle.clone(); + let address = record.address.clone(); + if !record.is_verified { + debug!( + "OpenSea search address({}) => {}={} not verified.", + address, sns_platform, sns_handle + ); + continue; + } + + if sns_platform == Platform::Unknown { + continue; + } + + if sns_platform != *platform { + // Do not push duplicate targets into fetchjob + next_targets.push(Target::Identity(sns_platform, sns_handle.clone())) + } + + let from = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Ethereum, + identity: address, + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let to = Identity { + uuid: Some(Uuid::new_v4()), + platform: sns_platform, + identity: sns_handle.clone(), + uid: None, + created_at: None, + display_name: Some(sns_handle.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + // add identity connected to hyper vertex + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &from, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &to, HYPER_EDGE), + )); + + let proof_forward = Proof { + uuid: Uuid::new_v4(), + source: DataSource::OpenSea, + level: ProofLevel::Confident, + record_id: None, + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::DataMgrService, + }; + + let proof_backward = Proof { + uuid: Uuid::new_v4(), + source: DataSource::OpenSea, + level: ProofLevel::Confident, + record_id: None, + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::DataMgrService, + }; + + let pf = proof_forward.wrapper(&from, &to, PROOF_EDGE); + let pb = proof_backward.wrapper(&to, &from, PROOF_REVERSE_EDGE); + + edges.push(EdgeWrapperEnum::new_proof_forward(pf)); + edges.push(EdgeWrapperEnum::new_proof_backward(pb)); + } + + Ok((next_targets, edges)) +} + +async fn search_opensea_account( + platform: &Platform, + identity: &str, +) -> Result, Error> { + let client = make_client(); + let uri: http::Uri = format!( + "{}/aggregation/opensea_account?platform={}&identity={}", + C.upstream.aggregation_service.url.clone(), + platform.to_string(), + identity + ) + .parse() + .map_err(|_err: InvalidUri| Error::ParamError(format!("OpenSea Uri format Error {}", _err)))?; + + let req = Request::builder() + .method(Method::GET) + .uri(uri) + .body(Body::empty()) + .map_err(|_err| { + Error::ParamError(format!("OpenSea search Build Request Error {}", _err)) + })?; + + let mut resp = request_with_timeout(&client, req, None) + .await + .map_err(|err| { + Error::ManualHttpClientError(format!("OpenSea search | error: {:?}", err.to_string())) + })?; + + if !resp.status().is_success() { + let err_message = format!("OpenSea search error, statusCode: {}", resp.status()); + error!(err_message); + return Err(Error::General(err_message, resp.status())); + } + + let result = match parse_body::(&mut resp).await { + Ok(result) => { + if result.code != 0 { + let err_message = format!( + "OpenSea search error | Code: {:?}, Message: {:?}", + result.code, result.msg + ); + error!(err_message); + return Err(Error::General( + err_message, + StatusCode::INTERNAL_SERVER_ERROR, + )); + } + let return_data: Vec = result.data.map_or(vec![], |res| res); + return_data + } + Err(err) => { + let err_message = format!("OpenSea search parse_body error: {:?}", err); + error!(err_message); + return Err(Error::General(err_message, resp.status())); + } + }; + + Ok(result) +} diff --git a/src/upstream/opensea/tests.rs b/src/upstream/opensea/tests.rs new file mode 100644 index 00000000..e69de29b diff --git a/src/upstream/proof_client/mod.rs b/src/upstream/proof_client/mod.rs index 11e3a37c..26bae60a 100644 --- a/src/upstream/proof_client/mod.rs +++ b/src/upstream/proof_client/mod.rs @@ -4,9 +4,12 @@ mod tests; use crate::config::C; use crate::error::Error; -use crate::tigergraph::edge::Proof; +use crate::tigergraph::edge::{ + HyperEdge, Proof, Wrapper, HYPER_EDGE, PROOF_EDGE, PROOF_REVERSE_EDGE, +}; use crate::tigergraph::upsert::create_identity_to_identity_proof_two_way_binding; -use crate::tigergraph::vertex::Identity; +use crate::tigergraph::vertex::{IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; use crate::upstream::{DataSource, Fetcher, Platform, ProofLevel, Target, TargetProcessedList}; use crate::util::make_http_client; use crate::util::{make_client, naive_now, parse_body, request_with_timeout, timestamp_to_naive}; @@ -73,6 +76,19 @@ impl Fetcher for ProofClient { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + match target { + Target::Identity(platform, identity) => { + batch_fetch_connections(platform, identity).await + } + Target::NFT(_, _, _, _) => todo!(), + } + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![ Platform::Ethereum, @@ -84,6 +100,158 @@ impl Fetcher for ProofClient { } } +#[tracing::instrument(level = "trace", fields(platform = %platform, identity = %identity))] +async fn batch_fetch_connections( + platform: &Platform, + identity: &str, +) -> Result<(TargetProcessedList, EdgeList), Error> { + let client = make_client(); + + let uri: http::Uri = format!( + "{}/v1/proof?exact=true&platform={}&identity={}", + C.upstream.proof_service.url, platform, identity + ) + .parse() + .map_err(|_err| Error::ParamError("Uri format Error".to_string()))?; + + let req = hyper::Request::builder() + .method(Method::GET) + .uri(uri) + .header("x-api-key", C.upstream.proof_service.api_key.clone()) + .body(Body::empty()) + .map_err(|_err| Error::ParamError(format!("Proof Service Build Request Error {}", _err)))?; + + let mut resp = request_with_timeout(&client, req, None) + .await + .map_err(|err| { + Error::ManualHttpClientError(format!( + "Proof Service fetch | error: {:?}", + err.to_string() + )) + })?; + + if !resp.status().is_success() { + let body: ErrorResponse = parse_body(&mut resp).await?; + error!("Proof Service fetch error, status {}", resp.status()); + return Err(Error::General( + format!("Proof Result Get Error: {}", body.message), + resp.status(), + )); + } + + let query_result: ProofQueryResponse = parse_body(&mut resp).await?; + if query_result.pagination.total == 0 { + error!("Proof Service ({}, {}) NoResult", platform, identity); + return Ok((vec![], vec![])); + } + + debug!(length = query_result.ids.len(), "Found."); + if query_result.ids.len() == 0 { + error!("Proof Service ({}, {}) NoResult", platform, identity); + return Ok((vec![], vec![])); + } + + let mut next_targets = TargetProcessedList::new(); + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + for id in query_result.ids { + let ProofPersona { avatar, proofs } = id; + + for p in proofs.into_iter() { + if p.is_valid == false { + continue; + } + let from: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::NextID, + identity: avatar.clone(), + uid: None, + created_at: timestamp_to_naive(p.created_at.to_string().parse::().unwrap(), 0), + display_name: Some(avatar.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let to_platform = Platform::from_str(p.platform.as_str()).unwrap_or(Platform::Unknown); + if to_platform == Platform::Unknown { + event!( + Level::WARN, + ?platform, + identity, + platform = p.platform, + "found unknown connected platform", + ); + continue; + } + + let to: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: to_platform, + identity: p.identity.to_string().to_lowercase(), + uid: None, + created_at: timestamp_to_naive(p.created_at.to_string().parse().unwrap(), 0), + // Don't use ETH's wallet as display_name, use ENS reversed lookup instead. + display_name: if to_platform == Platform::Ethereum { + None + } else { + Some(p.identity.clone()) + }, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let proof_forward: Proof = Proof { + uuid: Uuid::new_v4(), + source: DataSource::NextID, + level: ProofLevel::VeryConfident, + record_id: None, + created_at: timestamp_to_naive(p.created_at.to_string().parse().unwrap(), 0), + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + }; + + let proof_backward: Proof = Proof { + uuid: Uuid::new_v4(), + source: DataSource::NextID, + level: ProofLevel::VeryConfident, + record_id: None, + created_at: timestamp_to_naive(p.created_at.to_string().parse().unwrap(), 0), + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + }; + + // add identity connected to hyper vertex + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &from, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &to, HYPER_EDGE), + )); + + // two-way binding + let pf = proof_forward.wrapper(&from, &to, PROOF_EDGE); + let pb = proof_backward.wrapper(&to, &from, PROOF_REVERSE_EDGE); + + edges.push(EdgeWrapperEnum::new_proof_forward(pf)); + edges.push(EdgeWrapperEnum::new_proof_backward(pb)); + + next_targets.push(Target::Identity(to_platform, p.identity)); + } + } + next_targets.dedup(); + event!(Level::TRACE, "Next target count: {:?}", next_targets.len()); + Ok((next_targets, edges)) +} + #[tracing::instrument(level = "trace", fields(platform = %platform, identity = %identity))] async fn fetch_connections_by_platform_identity( platform: &Platform, diff --git a/src/upstream/rss3/mod.rs b/src/upstream/rss3/mod.rs index c474ffb6..911a9950 100644 --- a/src/upstream/rss3/mod.rs +++ b/src/upstream/rss3/mod.rs @@ -3,9 +3,10 @@ mod tests; use crate::config::C; use crate::error::Error; -use crate::tigergraph::edge::Hold; +use crate::tigergraph::edge::{Hold, HyperEdge, Wrapper, HOLD_CONTRACT, HYPER_EDGE}; use crate::tigergraph::upsert::create_identity_to_contract_hold_record; -use crate::tigergraph::vertex::{Contract, Identity}; +use crate::tigergraph::vertex::{Contract, IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; use crate::upstream::{ Chain, ContractCategory, DataSource, Fetcher, Platform, Target, TargetProcessedList, }; @@ -92,11 +93,195 @@ impl Fetcher for Rss3 { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + match target.platform()? { + Platform::Ethereum => batch_fetch_nfts(target).await, + _ => Ok((vec![], vec![])), + } + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::Ethereum]) } } +async fn batch_fetch_nfts(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let client = make_client(); + let address = target.identity()?.to_lowercase(); + let mut cursor = String::from(""); + + let mut next_targets = TargetProcessedList::new(); + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + loop { + let uri: http::Uri; + if cursor.len() == 0 { + uri = format!( + "{}/{}?tag=collectible&include_poap=true&refresh=true", + C.upstream.rss3_service.url, address + ) + .parse() + .map_err(|_err: InvalidUri| Error::ParamError(format!("Uri format Error {}", _err)))?; + } else { + uri = format!( + "{}/{}?tag=collectible&include_poap=true&refresh=true&cursor={}", + C.upstream.rss3_service.url, address, cursor + ) + .parse() + .map_err(|_err: InvalidUri| Error::ParamError(format!("Uri format Error {}", _err)))?; + } + + let req = hyper::Request::builder() + .method(Method::GET) + .uri(uri) + .body(Body::empty()) + .map_err(|_err| Error::ParamError(format!("Rss3 Build Request Error {}", _err)))?; + + let mut resp = request_with_timeout(&client, req, None) + .await + .map_err(|err| { + Error::ManualHttpClientError(format!( + "Rss3 fetch fetch | error: {:?}", + err.to_string() + )) + })?; + + let body: Rss3Response = parse_body(&mut resp).await?; + if body.total == 0 { + info!("Rss3 Response result is empty"); + // break; + } + + let result: Vec = body + .result + .into_iter() + .filter(|p| p.owner == address) + .collect(); + + for p in result.into_iter() { + if p.actions.len() == 0 { + continue; + } + + let found = p + .actions + .iter() + // collectible (transfer, mint, burn) share the same UMS, but approve/revoke not. + // we need to record is the `hold` relation, so burn is excluded + .filter(|a| { + (a.tag_type == "transfer" && p.tag_type == "transfer") + || (a.tag_type == "mint" && p.tag_type == "mint") + }) + .find(|a| (p.tag == "collectible" && a.tag == "collectible")); + + if found.is_none() { + continue; + } + let real_action = found.unwrap(); + + if real_action.metadata.symbol.is_none() + || real_action.metadata.symbol.as_ref().unwrap() == &String::from("ENS") + { + continue; + } + + let mut nft_category = ContractCategory::Unknown; + let standard = real_action.metadata.standard.clone(); + if let Some(standard) = standard { + if standard == "ERC-721".to_string() { + nft_category = ContractCategory::ERC721; + } else if standard == "ERC-1155".to_string() { + nft_category = ContractCategory::ERC1155; + } + } + if real_action.tag_type == "poap".to_string() { + nft_category = ContractCategory::POAP; + } + + let created_at_naive = match p.timestamp.as_ref() { + "" => None, + timestamp => match utc_to_naive(timestamp.to_string()) { + Ok(naive_dt) => Some(naive_dt), + Err(_) => None, // You may want to handle this error differently + }, + }; + + let from: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Ethereum, + identity: p.owner.to_lowercase(), + uid: None, + created_at: created_at_naive, + // Don't use ETH's wallet as display_name, use ENS reversed lookup instead. + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let chain = Chain::from_str(p.network.as_str()).unwrap_or_default(); + if chain == Chain::Unknown { + error!("Rss3 Fetch data | Unknown Chain, original data: {:?}", p); + continue; + } + let contract_addr = real_action + .metadata + .contract_address + .as_ref() + .unwrap() + .to_lowercase(); + let nft_id = real_action.metadata.id.as_ref().unwrap(); + + let to: Contract = Contract { + uuid: Uuid::new_v4(), + category: nft_category, + address: contract_addr.clone(), + chain, + symbol: Some(real_action.metadata.symbol.as_ref().unwrap().clone()), + updated_at: naive_now(), + }; + + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::Rss3, + transaction: Some(p.hash), + id: nft_id.clone(), + created_at: created_at_naive, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: None, + }; + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &from, HYPER_EDGE), + )); + let hdc = hold.wrapper(&from, &to, HOLD_CONTRACT); + edges.push(EdgeWrapperEnum::new_hold_contract(hdc)); + + next_targets.push(Target::NFT( + chain, + nft_category, + contract_addr.clone(), + nft_id.clone(), + )); + } + if body.cursor.is_none() || body.total < PAGE_LIMIT { + break; + } else { + cursor = body.cursor.unwrap(); + } + } + Ok((next_targets, edges)) +} + async fn fetch_nfts_by_account( _platform: &Platform, identity: &str, diff --git a/src/upstream/solana/mod.rs b/src/upstream/solana/mod.rs index 87058571..ef3f1446 100644 --- a/src/upstream/solana/mod.rs +++ b/src/upstream/solana/mod.rs @@ -2,16 +2,18 @@ mod tests; use crate::config::C; use crate::error::Error; -use crate::tigergraph::edge::Hold; -use crate::tigergraph::edge::Proof; -use crate::tigergraph::edge::Resolve; +use crate::tigergraph::edge::{ + Hold, HyperEdge, Proof, Resolve, Wrapper, HOLD_CONTRACT, HOLD_IDENTITY, HYPER_EDGE, PROOF_EDGE, + PROOF_REVERSE_EDGE, RESOLVE, REVERSE_RESOLVE, +}; use crate::tigergraph::upsert::create_ens_identity_ownership; use crate::tigergraph::upsert::create_identity_domain_resolve_record; use crate::tigergraph::upsert::create_identity_domain_reverse_resolve_record; use crate::tigergraph::upsert::create_identity_to_contract_hold_record; use crate::tigergraph::upsert::create_identity_to_identity_proof_two_way_binding; use crate::tigergraph::upsert::create_isolated_vertex; -use crate::tigergraph::vertex::{Contract, Identity}; +use crate::tigergraph::vertex::{Contract, IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; use crate::upstream::ProofLevel; use crate::upstream::{Chain, ContractCategory, DataFetcher, DataSource, DomainNameSystem}; use crate::util::{make_http_client, naive_now}; @@ -54,11 +56,424 @@ impl Fetcher for Solana { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + match target.platform()? { + Platform::Solana => batch_fetch_by_wallet(target).await, + Platform::SNS => batch_fetch_by_sns_handle(target).await, + Platform::Twitter => batch_fetch_by_twitter_handle(target).await, + _ => Ok((vec![], vec![])), + } + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::Solana, Platform::SNS, Platform::Twitter]) } } +async fn batch_fetch_by_wallet(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let owner: String = target.identity()?; + let rpc_client = get_rpc_client(C.upstream.solana_rpc.rpc_url.clone()); + let verified_owner = Pubkey::from_str(&owner)?; + let resolve_domains = fetch_resolve_domains(&rpc_client, &owner).await?; + + let mut next_targets = TargetProcessedList::new(); + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + let mut solana = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Solana, + identity: verified_owner.to_string(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: Some(format!( + "https://www.sns.id/profile?pubkey={}&subView=Show+All", + owner + )), + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &solana, HYPER_EDGE), + )); + + if resolve_domains.is_empty() { + trace!(?target, "Solana resolve domains is null"); + return Ok((vec![], edges)); + } + + let favourite_domain = fetch_register_favourite(&rpc_client, &owner).await?; + match favourite_domain { + Some(favourite_domain) => { + let format_sol = format_domain(&favourite_domain); + trace!(?target, "Favourite Domain Founded({})", format_sol); + solana.reverse = Some(true); // set reverse + solana.display_name = Some(format_sol.clone()); + let farvourite_sns = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::SNS, + identity: format_sol.clone(), + uid: None, + created_at: None, + display_name: Some(format_sol.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: Some(format!( + "https://www.sns.id/domain?domain={}", + favourite_domain + )), + updated_at: naive_now(), + expired_at: None, + reverse: Some(true), + }; + + let reverse: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::Solana, + system: DomainNameSystem::SNS, + name: format_sol.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + edges.push(EdgeWrapperEnum::new_hyper_edge(HyperEdge {}.wrapper( + &hv, + &farvourite_sns, + HYPER_EDGE, + ))); + let rr = reverse.wrapper(&solana, &farvourite_sns, REVERSE_RESOLVE); + edges.push(EdgeWrapperEnum::new_reverse_resolve(rr)); + } + None => trace!(?target, "Favourite Domain Not Set"), + }; + + let twitter_handle = get_handle_and_registry_key(&rpc_client, &owner).await?; + match twitter_handle { + Some(twitter_handle) => { + let format_twitter = twitter_handle.to_lowercase(); + let twitter = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Twitter, + identity: format_twitter.clone(), + uid: None, + created_at: None, + display_name: Some(format_twitter.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let proof_forward: Proof = Proof { + uuid: Uuid::new_v4(), + source: DataSource::SNS, + level: ProofLevel::VeryConfident, + record_id: None, + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + }; + + let proof_backward: Proof = Proof { + uuid: Uuid::new_v4(), + source: DataSource::SNS, + level: ProofLevel::VeryConfident, + record_id: None, + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + }; + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &twitter, HYPER_EDGE), + )); + let pf = proof_forward.wrapper(&solana, &twitter, PROOF_EDGE); + let pb = proof_backward.wrapper(&twitter, &solana, PROOF_REVERSE_EDGE); + + edges.push(EdgeWrapperEnum::new_proof_forward(pf)); + edges.push(EdgeWrapperEnum::new_proof_backward(pb)); + + next_targets.push(Target::Identity(Platform::Twitter, format_twitter.clone())) + } + None => trace!(?target, "Twitter Record Not Set"), + } + + trace!( + ?target, + domains = resolve_domains.len(), + "Solana Resolve Domains" + ); + for domain in resolve_domains.iter() { + let format_sol_handle = format_domain(domain); + let sns = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::SNS, + identity: format_sol_handle.clone(), + uid: None, + created_at: None, + display_name: Some(format_sol_handle.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: Some(format!("https://www.sns.id/domain?domain={}", domain)), + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::Solana, + transaction: Some("".to_string()), + id: format_sol_handle.clone(), + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: None, + }; + + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::Solana, + system: DomainNameSystem::SNS, + name: format_sol_handle.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + + let contract = Contract { + uuid: Uuid::new_v4(), + category: ContractCategory::SNS, + address: ContractCategory::SNS.default_contract_address().unwrap(), + chain: Chain::Solana, + symbol: Some("SNS".to_string()), + updated_at: naive_now(), + }; + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &sns, HYPER_EDGE), + )); + + // hold record + let hd = hold.wrapper(&solana, &sns, HOLD_IDENTITY); + let hdc = hold.wrapper(&solana, &contract, HOLD_CONTRACT); + // resolve record + let rs = resolve.wrapper(&sns, &solana, RESOLVE); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_hold_contract(hdc)); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + } + + Ok((next_targets, edges)) +} + +async fn batch_fetch_by_sns_handle( + target: &Target, +) -> Result<(TargetProcessedList, EdgeList), Error> { + let rpc_client = get_rpc_client(C.upstream.solana_rpc.rpc_url.clone()); + let name = target.identity()?; + let domain = trim_domain(name.clone()); + let owner = fetch_resolve_address(&rpc_client, &domain).await?; + + let mut next_targets = TargetProcessedList::new(); + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + match owner { + Some(owner) => { + let solana = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Solana, + identity: owner.to_string(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: Some(format!( + "https://www.sns.id/profile?pubkey={}&subView=Show+All", + owner.to_string() + )), + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let sns = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::SNS, + identity: name.clone(), + uid: None, + created_at: None, + display_name: Some(name.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: Some(format!("https://www.sns.id/domain?domain={}", name)), + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::Solana, + transaction: Some("".to_string()), + id: name.clone(), + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: None, + }; + + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::Solana, + system: DomainNameSystem::SNS, + name: name.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + + let contract = Contract { + uuid: Uuid::new_v4(), + category: ContractCategory::SNS, + address: ContractCategory::SNS.default_contract_address().unwrap(), + chain: Chain::Solana, + symbol: Some("SNS".to_string()), + updated_at: naive_now(), + }; + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &solana, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &sns, HYPER_EDGE), + )); + + // hold record + let hd = hold.wrapper(&solana, &sns, HOLD_IDENTITY); + let hdc = hold.wrapper(&solana, &contract, HOLD_CONTRACT); + // resolve record + let rs = resolve.wrapper(&sns, &solana, RESOLVE); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_hold_contract(hdc)); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + + next_targets.push(Target::Identity(Platform::Solana, owner.to_string())); + } + None => trace!(?target, "Owner not found"), + } + + Ok((next_targets, edges)) +} + +async fn batch_fetch_by_twitter_handle( + target: &Target, +) -> Result<(TargetProcessedList, EdgeList), Error> { + let rpc_client = get_rpc_client(C.upstream.solana_rpc.rpc_url.clone()); + let twitter_handle = target.identity()?; + + let mut next_targets = TargetProcessedList::new(); + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + let solana_wallet = get_twitter_registry(&rpc_client, &twitter_handle).await?; + match solana_wallet { + Some(solana_wallet) => { + trace!( + ?target, + "Solana Wallet Founded by Twitter({}): {}", + twitter_handle, + solana_wallet.to_string() + ); + + let solana = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Solana, + identity: solana_wallet.to_string(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: Some(format!( + "https://www.sns.id/profile?pubkey={}&subView=Show+All", + solana_wallet.to_string() + )), + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let format_twitter = twitter_handle.to_lowercase(); + let twitter = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Twitter, + identity: format_twitter.clone(), + uid: None, + created_at: None, + display_name: Some(format_twitter.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let proof_forward: Proof = Proof { + uuid: Uuid::new_v4(), + source: DataSource::SNS, + level: ProofLevel::VeryConfident, + record_id: None, + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + }; + + let proof_backward: Proof = Proof { + uuid: Uuid::new_v4(), + source: DataSource::SNS, + level: ProofLevel::VeryConfident, + record_id: None, + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + }; + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &solana, HYPER_EDGE), + )); + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &twitter, HYPER_EDGE), + )); + let pf = proof_forward.wrapper(&solana, &twitter, PROOF_EDGE); + let pb = proof_backward.wrapper(&twitter, &solana, PROOF_REVERSE_EDGE); + + edges.push(EdgeWrapperEnum::new_proof_forward(pf)); + edges.push(EdgeWrapperEnum::new_proof_backward(pb)); + + next_targets.push(Target::Identity( + Platform::Solana, + solana_wallet.to_string(), + )); + } + None => trace!(?target, "Solana Wallet Not Found"), + } + + Ok((next_targets, edges)) +} + async fn fetch_by_wallet(target: &Target) -> Result { let mut next_targets: TargetProcessedList = Vec::new(); let rpc_client = get_rpc_client(C.upstream.solana_rpc.rpc_url.clone()); diff --git a/src/upstream/space_id/mod.rs b/src/upstream/space_id/mod.rs index d69765d5..2996ff46 100644 --- a/src/upstream/space_id/mod.rs +++ b/src/upstream/space_id/mod.rs @@ -2,11 +2,14 @@ mod tests; use crate::config::C; use crate::error::Error; -use crate::tigergraph::edge::{Hold, Resolve}; +use crate::tigergraph::edge::{ + Hold, HyperEdge, Resolve, Wrapper, HOLD_IDENTITY, HYPER_EDGE, RESOLVE, REVERSE_RESOLVE, +}; use crate::tigergraph::upsert::create_identity_domain_resolve_record; use crate::tigergraph::upsert::create_identity_domain_reverse_resolve_record; use crate::tigergraph::upsert::create_identity_to_identity_hold_record; -use crate::tigergraph::vertex::Identity; +use crate::tigergraph::vertex::{IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; use crate::upstream::{ DataFetcher, DataSource, DomainNameSystem, Fetcher, Platform, Target, TargetProcessedList, }; @@ -53,11 +56,207 @@ impl Fetcher for SpaceId { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + match target.platform()? { + Platform::Ethereum => batch_fetch_by_wallet(target).await, + Platform::SpaceId => batch_fetch_by_handle(target).await, + _ => Ok((vec![], vec![])), + } + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::SpaceId, Platform::Ethereum]) } } +async fn batch_fetch_by_wallet(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let identity = target.identity()?; + let name = get_name(&identity).await?; + if name.is_none() { + // name=null, address does not have a valid primary name + return Ok((vec![], vec![])); + } + + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + let addr: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Ethereum, + identity: identity.to_lowercase(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(true), + }; + + let sid: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::SpaceId, + identity: name.clone().unwrap(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(true), + }; + + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::SpaceId, + transaction: None, + id: "".to_string(), + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: None, + }; + + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::SpaceId, + system: DomainNameSystem::SpaceId, + name: name.clone().unwrap(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + let reverse: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::SpaceId, + system: DomainNameSystem::SpaceId, + name: name.clone().unwrap(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &addr, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &sid, HYPER_EDGE), + )); + + let rrs = reverse.wrapper(&addr, &sid, REVERSE_RESOLVE); + let hd = hold.wrapper(&addr, &sid, HOLD_IDENTITY); + let rs = resolve.wrapper(&sid, &addr, RESOLVE); + + // hold record + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + // 'regular' resolution involves mapping from a name to an address. + edges.push(EdgeWrapperEnum::new_resolve(rs)); + // 'reverse' resolution maps from an address back to a name. + edges.push(EdgeWrapperEnum::new_reverse_resolve(rrs)); + + Ok((vec![], edges)) +} + +async fn batch_fetch_by_handle(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let name = target.identity()?; + let address = get_address(&name).await?; + + let mut next_targets = TargetProcessedList::new(); + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + let mut addr: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Ethereum, + identity: address.clone().to_lowercase(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let mut sid: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::SpaceId, + identity: name.clone(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::SpaceId, + transaction: Some("".to_string()), + id: "".to_string(), + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: None, + }; + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::SpaceId, + system: DomainNameSystem::SpaceId, + name: name.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + // lookup reverse resolve name + if let Some(domain) = get_name(&address).await? { + // 'reverse' resolution maps from an address back to a name. + let reverse: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::SpaceId, + system: DomainNameSystem::SpaceId, + name: domain, + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + addr.reverse = Some(true); + sid.reverse = Some(true); + let rrs = reverse.wrapper(&addr, &sid, REVERSE_RESOLVE); + edges.push(EdgeWrapperEnum::new_reverse_resolve(rrs)); + } + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &addr, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &sid, HYPER_EDGE), + )); + + // hold record + let hd = hold.wrapper(&addr, &sid, HOLD_IDENTITY); + // 'regular' resolution involves mapping from a name to an address. + let rs = resolve.wrapper(&sid, &addr, RESOLVE); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + + next_targets.push(Target::Identity( + Platform::Ethereum, + address.clone().to_lowercase(), + )); + Ok((next_targets, edges)) +} + async fn fetch_connections_by_platform_identity( platform: &Platform, identity: &str, diff --git a/src/upstream/sybil_list/mod.rs b/src/upstream/sybil_list/mod.rs index e09ae04b..83923e7a 100644 --- a/src/upstream/sybil_list/mod.rs +++ b/src/upstream/sybil_list/mod.rs @@ -7,6 +7,7 @@ use crate::error::Error; use crate::tigergraph::edge::Proof; use crate::tigergraph::upsert::create_identity_to_identity_proof_two_way_binding; use crate::tigergraph::vertex::Identity; +use crate::tigergraph::EdgeList; use crate::upstream::{DataSource, Fetcher, Platform, ProofLevel, TargetProcessedList}; use crate::util::make_http_client; use crate::util::{make_client, naive_now, parse_body, request_with_timeout, timestamp_to_naive}; @@ -187,6 +188,15 @@ impl Fetcher for SybilList { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + // TODO: prefetch: move this logic to `data_process` module as a scheduled asynchronous fetch + Ok((vec![], vec![])) + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::Ethereum, Platform::Twitter]) } diff --git a/src/upstream/tests.rs b/src/upstream/tests.rs index 9c2156fe..2d90e484 100644 --- a/src/upstream/tests.rs +++ b/src/upstream/tests.rs @@ -1,5 +1,7 @@ use crate::error::Error; -use crate::upstream::{fetch_all, fetch_one, Chain, ContractCategory, Platform, Target}; +use crate::upstream::{ + batch_fetch_upstream, fetch_all, fetch_one, Chain, ContractCategory, Platform, Target, +}; #[tokio::test] async fn test_fetch_one_result() -> Result<(), Error> { @@ -9,17 +11,81 @@ async fn test_fetch_one_result() -> Result<(), Error> { Ok(()) } +#[tokio::test] +async fn test_batch_fetch_upstream() -> Result<(), Error> { + let target = Target::Identity(Platform::Dotbit, "threebody.bit".into()); + + let result = batch_fetch_upstream(&target).await?; + println!("{:?}", result); + Ok(()) +} + #[tokio::test] async fn test_fetch_all() -> Result<(), Error> { + // fetch_all( + // vec![Target::Identity( + // Platform::Ethereum, + // "0x0da0ee86269797618032e56a69b1aad095c581fc".into(), + // )], + // Some(5), + // ) + // .await?; + + // fetch_all( + // vec![Target::Identity( + // Platform::Ethereum, + // "0x934b510d4c9103e6a87aef13b816fb080286d649".into(), + // )], + // Some(5), + // ) + // .await?; + fetch_all( vec![Target::Identity( Platform::Ethereum, - "0x934b510d4c9103e6a87aef13b816fb080286d649".into(), + "0xd8da6bf26964af9d7eed9e03e53415d37aa96045".into(), )], Some(5), ) .await?; + // fetch_all( + // vec![Target::NFT( + // Chain::Ethereum, + // ContractCategory::ENS, + // ContractCategory::ENS.default_contract_address().unwrap(), + // "niconico.eth".to_string(), + // )], + // Some(5), + // ) + // .await?; + + // fetch_all( + // vec![Target::NFT( + // Chain::Ethereum, + // ContractCategory::ENS, + // ContractCategory::ENS.default_contract_address().unwrap(), + // "vitalik.eth".to_string(), + // )], + // Some(5), + // ) + // .await?; + + // fetch_all( + // vec![Target::Identity(Platform::CKB, "ckb1qzfhdsa4syv599s2s3nfrctwga70g0tu07n9gpnun9ydlngf5vsnwqggq7v6mzt3n8wv9y2n6h9z429ta0auek7v05yq0xdd39cenhxzj9fatj324z47h77vm0x869nu03m".into())], + // Some(5), + // ) + // .await?; + + // fetch_all( + // vec![Target::Identity( + // Platform::Ethereum, + // "0x6d910bea79aaf318e7170c6fb8318d9c466b2164".into(), + // )], + // Some(5), + // ) + // .await?; + Ok(()) } diff --git a/src/upstream/the_graph/mod.rs b/src/upstream/the_graph/mod.rs index 02eaa04a..d125ea96 100644 --- a/src/upstream/the_graph/mod.rs +++ b/src/upstream/the_graph/mod.rs @@ -3,12 +3,16 @@ mod tests; use crate::config::C; use crate::error::Error; -use crate::tigergraph::edge::{Hold, Resolve}; +use crate::tigergraph::edge::{ + Hold, HyperEdge, Resolve, Wrapper, HOLD_CONTRACT, HOLD_IDENTITY, HYPER_EDGE, RESOLVE, + RESOLVE_CONTRACT, +}; use crate::tigergraph::upsert::create_contract_to_identity_resolve_record; -use crate::tigergraph::upsert::create_ens_identity_ownership; use crate::tigergraph::upsert::create_identity_domain_resolve_record; use crate::tigergraph::upsert::create_identity_to_contract_hold_record; -use crate::tigergraph::vertex::{Contract, Identity}; +use crate::tigergraph::upsert::{create_ens_identity_ownership, create_ens_identity_resolve}; +use crate::tigergraph::vertex::{Contract, IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; use crate::upstream::{ Chain, ContractCategory, DataFetcher, DataSource, DomainNameSystem, Fetcher, Platform, Target, TargetProcessedList, @@ -179,6 +183,14 @@ impl Fetcher for TheGraph { perform_fetch(target).await } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + batch_perform_fetch(target).await + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::Ethereum]) || target.in_nft_supported(vec![ContractCategory::ENS], vec![Chain::Ethereum]) @@ -257,6 +269,171 @@ async fn fetch_domains(target: &Target) -> Result, Error> { Ok(merged_domains) } +async fn batch_perform_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let merged_domains = fetch_domains(target).await?; + if merged_domains.is_empty() { + info!(?target, "TheGraph: No result"); + return Ok((vec![], vec![])); + } + + let platform = match target { + Target::Identity(_platform, _identity) => *_platform, + Target::NFT(_chain, _category, _contract_addr, _ens_name) => Platform::ENS, + }; + + let mut next_targets: TargetProcessedList = vec![]; + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + for domain in merged_domains.into_iter() { + let creation_tx = domain + .events + .first() + .map(|event| event.transaction_id.clone()); + let ens_created_at = parse_timestamp(&domain.created_at).ok(); + let ens_expired_at = match &domain.registration { + Some(registration) => parse_timestamp(®istration.expiry_date).ok(), + None => None, + }; + + let owner = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Ethereum, + identity: domain.owner.id.clone(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + let ens_domain: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::ENS, + identity: domain.name.clone(), + uid: None, + created_at: ens_created_at, + display_name: Some(domain.name.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: ens_expired_at, + reverse: Some(false), + }; + let contract = Contract { + uuid: Uuid::new_v4(), + category: ContractCategory::ENS, + address: ContractCategory::ENS.default_contract_address().unwrap(), + chain: Chain::Ethereum, + symbol: None, + updated_at: naive_now(), + }; + let ownership: Hold = Hold { + uuid: Uuid::new_v4(), + transaction: creation_tx, + id: domain.name.clone(), + source: DataSource::TheGraph, + created_at: ens_created_at, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: ens_expired_at, + }; + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &owner, HYPER_EDGE), + )); + + let resolved_address = domain.resolved_address.map(|r| r.id); + match resolved_address.clone() { + None => { + // Resolve record not existed anymore. Save owner address. + trace!( + ?target, + "TheGraph: Resolve address not existed. Save owner address" + ); + // hold record + let hd = ownership.wrapper(&owner, &ens_domain, HOLD_IDENTITY); + let hdc = ownership.wrapper(&owner, &contract, HOLD_CONTRACT); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_hold_contract(hdc)); + // Append up_next + if platform == Platform::ENS { + next_targets.push(Target::Identity( + Platform::Ethereum, + domain.owner.id.clone(), + )); + } + } + Some(address) => { + // Filter zero address (without last 4 digits) + if !address.starts_with("0x000000000000000000000000000000000000") { + // Create resolve record + debug!(?target, address, domain = domain.name, "TheGraph: Resolved"); + let resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::TheGraph, + system: DomainNameSystem::ENS, + name: domain.name.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + + let domain_owner = domain.owner.id.clone(); + + if address == domain_owner { + // ens_domain will be added to hyper_vertex IdentitiesGraph + // only when resolvedAddress == owner + edges.push(EdgeWrapperEnum::new_hyper_edge(HyperEdge {}.wrapper( + &hv, + &ens_domain, + HYPER_EDGE, + ))); + + // hold record + let hd = ownership.wrapper(&owner, &ens_domain, HOLD_IDENTITY); + let hdc = ownership.wrapper(&owner, &contract, HOLD_CONTRACT); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_hold_contract(hdc)); + + // resolve record + let rs = resolve.wrapper(&ens_domain, &owner, RESOLVE); + let rsc = resolve.wrapper(&contract, &owner, RESOLVE_CONTRACT); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + edges.push(EdgeWrapperEnum::new_resolve_contract(rsc)); + + // Append up_next + if platform == Platform::ENS { + next_targets + .push(Target::Identity(Platform::Ethereum, domain_owner.clone())); + } + } else { + debug!( + ?target, + address, domain_owner, "TheGraph: address != domain_owner" + ); + // hold record + let hd = ownership.wrapper(&owner, &ens_domain, HOLD_IDENTITY); + let hdc = ownership.wrapper(&owner, &contract, HOLD_CONTRACT); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_hold_contract(hdc)); + // Append up_next + if platform == Platform::ENS { + next_targets + .push(Target::Identity(Platform::Ethereum, domain_owner.clone())); + } + } + } + } + } + } + next_targets.dedup(); + return Ok((next_targets, edges)); +} + /// Focus on `ENS Domains` Saveing. async fn perform_fetch(target: &Target) -> Result { let merged_domains = fetch_domains(target).await?; @@ -331,6 +508,10 @@ async fn perform_fetch(target: &Target) -> Result { None => { // Resolve record not existed anymore. Maybe deleted by user. trace!(?target, "TheGraph: Resolve address not existed."); + // Save owner address + create_ens_identity_ownership(&cli, &owner, &ens_domain, &ownership).await?; + create_identity_to_contract_hold_record(&cli, &owner, &contract, &ownership) + .await?; } Some(address) => { // Filter zero address (without last 4 digits) @@ -361,30 +542,57 @@ async fn perform_fetch(target: &Target) -> Result { updated_at: naive_now(), }; - // regular resolved address - create_identity_domain_resolve_record( - &cli, - &ens_domain, - &resolve_target, - &resolve, - ) - .await?; - create_contract_to_identity_resolve_record( - &cli, - &contract, - &resolve_target, - &resolve, - ) - .await?; + let domain_owner = domain.owner.id.clone(); + if address != domain_owner { + // Save hold, resolve but not connect IdentityGraph HyperVetex + // create owner-hold-ens(contract) + create_ens_identity_ownership(&cli, &owner, &ens_domain, &ownership) + .await?; + create_identity_to_contract_hold_record( + &cli, &owner, &contract, &ownership, + ) + .await?; + + // create ens-resolve-address(contract) + create_ens_identity_resolve(&cli, &ens_domain, &resolve_target, &resolve) + .await?; + create_contract_to_identity_resolve_record( + &cli, + &contract, + &resolve_target, + &resolve, + ) + .await?; + } else { + // Save ens_identity as Identity in IdentityGraph + // create owner-hold-ens(contract) + create_ens_identity_ownership(&cli, &owner, &ens_domain, &ownership) + .await?; + create_identity_to_contract_hold_record( + &cli, &owner, &contract, &ownership, + ) + .await?; + + // create ens-resolve-addres(IdentityGraph) + create_identity_domain_resolve_record( + &cli, + &ens_domain, + &resolve_target, + &resolve, + ) + .await?; + create_contract_to_identity_resolve_record( + &cli, + &contract, + &resolve_target, + &resolve, + ) + .await?; + } } } } - // create owner-hold-ens(contract) - // ENS ownership create `Hold_Identity` connection but only EvmAddress connected to HyperVertex - create_ens_identity_ownership(&cli, &owner, &ens_domain, &ownership).await?; - create_identity_to_contract_hold_record(&cli, &owner, &contract, &ownership).await?; - // Append up_next match target { Target::Identity(_, _) => next_targets.push(Target::NFT( diff --git a/src/upstream/types/data_source.rs b/src/upstream/types/data_source.rs index 4fc89241..6ebf0831 100644 --- a/src/upstream/types/data_source.rs +++ b/src/upstream/types/data_source.rs @@ -184,7 +184,7 @@ pub enum DataSource { /// Twitter <-> Ethereum /// Manually added by Firefly.land team. /// Cannot be verified by third party, only trust the team. - #[strum(serialize = "hand_writing")] + #[strum(serialize = "manually_added")] #[serde(rename = "manually_added")] #[graphql(name = "manually_added")] ManuallyAdded, diff --git a/src/upstream/types/platform.rs b/src/upstream/types/platform.rs index dcde9e0b..136e6971 100644 --- a/src/upstream/types/platform.rs +++ b/src/upstream/types/platform.rs @@ -74,6 +74,12 @@ pub enum Platform { #[graphql(name = "facebook")] Facebook, + /// Instagram + #[strum(serialize = "instagram")] + #[serde(rename = "instagram")] + #[graphql(name = "instagram")] + Instagram, + /// Mastodon maintained by Sujitech #[strum(serialize = "mstdnjp")] #[serde(rename = "mstdnjp")] @@ -206,6 +212,7 @@ impl From for DomainNameSystem { Platform::Crossbell => DomainNameSystem::SpaceId, Platform::ENS => DomainNameSystem::ENS, Platform::SNS => DomainNameSystem::SNS, + Platform::Genome => DomainNameSystem::Genome, _ => DomainNameSystem::Unknown, } } diff --git a/src/upstream/unstoppable/mod.rs b/src/upstream/unstoppable/mod.rs index bfe7353c..9bb7266f 100644 --- a/src/upstream/unstoppable/mod.rs +++ b/src/upstream/unstoppable/mod.rs @@ -2,18 +2,18 @@ mod tests; use crate::config::C; use crate::error::Error; -use crate::tigergraph::edge::Hold; -use crate::tigergraph::edge::Resolve; +use crate::tigergraph::edge::{ + Hold, HyperEdge, Resolve, Wrapper, HOLD_IDENTITY, HYPER_EDGE, RESOLVE, REVERSE_RESOLVE, +}; use crate::tigergraph::upsert::create_identity_domain_resolve_record; use crate::tigergraph::upsert::create_identity_domain_reverse_resolve_record; use crate::tigergraph::upsert::create_identity_to_identity_hold_record; -use crate::tigergraph::vertex::Identity; -// use crate::graph::{new_db_connection, vertex::Identity}; +use crate::tigergraph::vertex::{IdentitiesGraph, Identity}; +use crate::tigergraph::{EdgeList, EdgeWrapperEnum}; use crate::upstream::{ DataFetcher, DataSource, DomainNameSystem, Fetcher, Platform, Target, TargetProcessedList, }; use crate::util::{make_client, make_http_client, naive_now, parse_body, request_with_timeout}; -// use aragog::DatabaseConnection; use async_trait::async_trait; use futures::future::join_all; use http::uri::InvalidUri; @@ -113,11 +113,232 @@ impl Fetcher for UnstoppableDomains { } } + async fn batch_fetch(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + if !Self::can_fetch(target) { + return Ok((vec![], vec![])); + } + + match target.platform()? { + Platform::Ethereum => batch_fetch_by_wallet(target).await, + Platform::UnstoppableDomains => batch_fetch_by_handle(target).await, + _ => Ok((vec![], vec![])), + } + } + fn can_fetch(target: &Target) -> bool { target.in_platform_supported(vec![Platform::UnstoppableDomains, Platform::Ethereum]) } } +async fn batch_fetch_by_wallet(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let address = target.identity()?.to_lowercase(); + + let mut cnt: u32 = 0; + let mut next = String::from(""); + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + while cnt < u32::MAX { + let result = fetch_domain(&address, &next).await?; + cnt += result.data.len() as u32; + + for item in result.data.iter() { + let mut addr: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Ethereum, + identity: address.clone(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let mut ud: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::UnstoppableDomains, + identity: item.id.clone(), + uid: None, + created_at: None, + display_name: Some(item.attributes.meta.domain.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::UnstoppableDomains, + transaction: Some("".to_string()), + id: item + .attributes + .meta + .token_id + .clone() + .unwrap_or("".to_string()), + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: None, + }; + + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::UnstoppableDomains, + system: DomainNameSystem::UnstoppableDomains, + name: item.id.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + + if item.attributes.meta.reverse { + // reverse = true + // 'reverse' resolution maps from an address back to a name. + let reverse: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::UnstoppableDomains, + system: DomainNameSystem::UnstoppableDomains, + name: item.id.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + addr.reverse = Some(true); + ud.reverse = Some(false); + let rrs = reverse.wrapper(&addr, &ud, REVERSE_RESOLVE); + edges.push(EdgeWrapperEnum::new_reverse_resolve(rrs)); + } + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &addr, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &ud, HYPER_EDGE), + )); + + let hd = hold.wrapper(&addr, &ud, HOLD_IDENTITY); + let rs = resolve.wrapper(&ud, &addr, RESOLVE); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + } + + if result.meta.has_more { + next = result.meta.next; + } else { + break; + } + } + Ok((vec![], edges)) +} + +async fn batch_fetch_by_handle(target: &Target) -> Result<(TargetProcessedList, EdgeList), Error> { + let name = target.identity()?; + let result = fetch_owner(&name).await?; + if result.meta.owner.is_none() { + warn!("UnstoppableDomains target {} | No Result", target); + return Ok((vec![], vec![])); + } + + if result.meta.owner.clone().unwrap().to_lowercase() == UNKNOWN_OWNER { + warn!("UnstoppableDomains owner is zero address {}", target); + return Ok((vec![], vec![])); + } + + let mut next_targets = TargetProcessedList::new(); + let mut edges = EdgeList::new(); + let hv = IdentitiesGraph::default(); + + let mut addr: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::Ethereum, + identity: result.meta.owner.clone().unwrap().to_lowercase(), + uid: None, + created_at: None, + display_name: None, + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let mut ud: Identity = Identity { + uuid: Some(Uuid::new_v4()), + platform: Platform::UnstoppableDomains, + identity: result.meta.domain.clone(), + uid: None, + created_at: None, + display_name: Some(result.meta.domain.clone()), + added_at: naive_now(), + avatar_url: None, + profile_url: None, + updated_at: naive_now(), + expired_at: None, + reverse: Some(false), + }; + + let hold: Hold = Hold { + uuid: Uuid::new_v4(), + source: DataSource::UnstoppableDomains, + transaction: None, + id: result.meta.token_id.unwrap_or("".to_string()), + created_at: None, + updated_at: naive_now(), + fetcher: DataFetcher::RelationService, + expired_at: None, + }; + + let resolve: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::UnstoppableDomains, + system: DomainNameSystem::UnstoppableDomains, + name: result.meta.domain.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + + if result.meta.reverse { + // reverse = true + // 'reverse' resolution maps from an address back to a name. + let reverse: Resolve = Resolve { + uuid: Uuid::new_v4(), + source: DataSource::UnstoppableDomains, + system: DomainNameSystem::UnstoppableDomains, + name: result.meta.domain.clone(), + fetcher: DataFetcher::RelationService, + updated_at: naive_now(), + }; + addr.reverse = Some(true); + ud.reverse = Some(true); + let rrs = reverse.wrapper(&addr, &ud, REVERSE_RESOLVE); + edges.push(EdgeWrapperEnum::new_reverse_resolve(rrs)); + } + + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &addr, HYPER_EDGE), + )); + edges.push(EdgeWrapperEnum::new_hyper_edge( + HyperEdge {}.wrapper(&hv, &ud, HYPER_EDGE), + )); + + let hd = hold.wrapper(&addr, &ud, HOLD_IDENTITY); + let rs = resolve.wrapper(&ud, &addr, RESOLVE); + edges.push(EdgeWrapperEnum::new_hold_identity(hd)); + edges.push(EdgeWrapperEnum::new_resolve(rs)); + + next_targets.push(Target::Identity( + Platform::Ethereum, + result.meta.owner.clone().unwrap().to_lowercase(), + )); + + Ok((next_targets, edges)) +} + async fn fetch_connections_by_platform_identity( platform: &Platform, identity: &str, @@ -167,19 +388,23 @@ async fn fetch_domain(owners: &str, page: &str) -> Result(&mut body).await { Ok(result) => result, Err(_) => { - let err: BadResponse = parse_body(&mut body).await?; - let err_message = format!( - "UnstoppableDomains fetch error, Code: {}, Message: {}", - err.code, err.message - ); - error!(err_message); - return Err(Error::General( - err_message, - lambda_http::http::StatusCode::INTERNAL_SERVER_ERROR, - )); + match parse_body::(&mut body).await { + Ok(bad) => { + let err_message = format!( + "UnstoppableDomains fetch error, Code: {}, Message: {}", + bad.code, bad.message + ); + error!(err_message); + return Err(Error::General( + err_message, + lambda_http::http::StatusCode::INTERNAL_SERVER_ERROR, + )); + } + Err(err) => return Err(err), + }; } }; Ok(result) @@ -370,16 +595,19 @@ async fn fetch_owner(domains: &str) -> Result { let result = match parse_body::(&mut resp).await { Ok(result) => result, - Err(_) => { - let err: BadResponse = parse_body(&mut resp).await?; - let err_message = format!( - "UnstoppableDomains fetch | errCode: {}, errMessage: {}", - err.code, err.message - ); - error!(err_message); - return Err(Error::General(err_message, resp.status())); - } + Err(_) => match parse_body::(&mut resp).await { + Ok(bad) => { + let err_message = format!( + "UnstoppableDomains fetch | errCode: {}, errMessage: {}", + bad.code, bad.message + ); + error!(err_message); + return Err(Error::General(err_message, resp.status())); + } + Err(err) => return Err(err), + }, }; + Ok(result) }