Add documentation for async-usercalls and address other minor issues

intel-sgx/async-usercalls/src/lib.rs

@@ -1,3 +1,39 @@
+//! This crate provides an interface for performing asynchronous usercalls in
+//! SGX enclaves. The motivation behind asynchronous usercalls and ABI
+//! documentation can be found
+//! [here](https://edp.fortanix.com/docs/api/fortanix_sgx_abi/async/index.html).
+//! The API provided here is fairly low level and is not meant for general use.
+//! These APIs can be used to implement [mio] abstractions which in turn
+//! allows us to use [tokio] in SGX enclaves!
+//!
+//! The main interface is provided through `AsyncUsercallProvider` which works
+//! in tandem with `CallbackHandler`:
+//! ```
+//! use async_usercalls::AsyncUsercallProvider;
+//! use std::{io::Result, net::TcpStream, sync::mpsc, time::Duration};
+//!
+//! let (provider, callback_handler) = AsyncUsercallProvider::new();
+//! let (tx, rx) = mpsc::sync_channel(1);
+//! // The closure is called when userspace sends back the result of the
+//! // usercall.
+//! let cancel_handle = provider.connect_stream("www.example.com:80", move |res| {
+//!     tx.send(res).unwrap();
+//! });
+//! // We can cancel the connect usercall using `cancel_handle.cancel()`, but
+//! // note that we may still get a successful result.
+//! // We need to poll `callback_handler` to make progress.
+//! loop {
+//!     let n = callback_handler.poll(Some(Duration::from_millis(100)));
+//!     if n > 0 {
+//!         break; // at least 1 callback function was executed!
+//!     }
+//! }
+//! let connect_result: Result<TcpStream> = rx.recv().unwrap();
+//! ```
+//!
+//! [mio]: https://docs.rs/mio/latest/mio/
+//! [tokio]: https://docs.rs/tokio/latest/tokio/
+
 #![feature(sgx_platform)]
 #![feature(never_type)]
 #![cfg_attr(test, feature(unboxed_closures))]

intel-sgx/async-usercalls/src/provider_core.rs

@@ -64,3 +64,13 @@ impl Drop for ProviderCore {
         PROVIDERS.remove_provider(self.provider_id);
     }
 }
+
+pub trait ProviderId {
+    fn provider_id(&self) -> u32;
+}
+
+impl<T> ProviderId for Identified<T> {
+    fn provider_id(&self) -> u32 {
+        (self.id >> 32) as u32
+    }
+}

intel-sgx/async-usercalls/src/queues.rs

@@ -1,4 +1,5 @@
 use crate::hacks::{alloc_descriptor, async_queues, to_enclave, Cancel, Return, Usercall};
+use crate::provider_core::ProviderId;
 use crossbeam_channel as mpmc;
 use fortanix_sgx_abi::{EV_CANCELQ_NOT_FULL, EV_RETURNQ_NOT_EMPTY, EV_USERCALLQ_NOT_FULL};
 use ipc_queue::{self, Identified, QueueEvent, RecvError, SynchronizationError, Synchronizer};
@@ -89,10 +90,9 @@ impl ReturnHandler {
         // taking the lock should be fast.
         let provider_map = self.provider_map.lock().unwrap();
         for ret in returns {
-            let provider_id = (ret.id >> 32) as u32;
             // NOTE: some providers might decide not to receive results of usercalls they send
             // because the results are not interesting, e.g. BatchDropProvider.
-            if let Some(sender) = provider_map.get(provider_id).and_then(|entry| entry.as_ref()) {
+            if let Some(sender) = provider_map.get(ret.provider_id()).and_then(|entry| entry.as_ref()) {
                 let _ = sender.send(*ret);
             }
         }
@@ -101,6 +101,8 @@ impl ReturnHandler {
     fn run(self) {
         let mut returns = [Identified::default(); Self::RECV_BATCH_SIZE];
         loop {
+            // Block until there is a return. Then we receive any other values
+            // from the return queue **without** blocking using `try_iter()`.
             let first = match self.return_queue_rx.recv() {
                 Ok(ret) => ret,
                 Err(RecvError::Closed) => break,

intel-sgx/enclave-runner/src/usercalls/mod.rs

@@ -47,6 +47,10 @@ lazy_static! {
     static ref DEBUGGER_TOGGLE_SYNC: Mutex<()> = Mutex::new(());
 }
 
+// This is not an event in the sense that it could be passed to `send()` or
+// `wait()` usercalls in enclave code. However, it's easier for the enclave
+// runner implementation to lump it in with events. Also note that this constant
+// is not public.
 const EV_ABORT: u64 = 0b0000_0000_0001_0000;
 
 const USERCALL_QUEUE_SIZE: usize = 16;