Update benchmark implementations

weiznich · weiznich · commit 9358d4ea5f20 · 2021-11-11T15:17:59.000+01:00
* Use the tokio runtime for sqlx/sea-orm as requested here: transact-rs/sqlx#1481 (comment) * Update quaint to a version that uses tokio 1.0 and reenable it in the metrics job
diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml
@@ -69,21 +69,21 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: bench
-          args: --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}} sqlx-bench sqlx/${{matrix.backend}} async-std rust_postgres futures sea-orm sea-orm/sqlx-${{matrix.backend}} criterion/async_std"
+          args: --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}} sqlx-bench sqlx/${{matrix.backend}} rust_postgres futures sea-orm sea-orm/sqlx-${{matrix.backend}} criterion/async_tokio quaint quaint/postgresql quaint/serde-support serde"
 
       - name: Run Benchmarks (Sqlite)
         if: matrix.backend == 'sqlite'
         uses: actions-rs/cargo@v1
         with:
           command: bench
-          args: --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}} sqlx-bench sqlx/${{matrix.backend}} async-std rusqlite futures sea-orm sea-orm/sqlx-${{matrix.backend}} criterion/async_std"
+          args: --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}} sqlx-bench sqlx/${{matrix.backend}} tokio rusqlite futures sea-orm sea-orm/sqlx-${{matrix.backend}} criterion/async_tokio quaint quaint/sqlite quaint/serde-support serde"
 
       - name: Run Benchmarks (Mysql)
         if: matrix.backend == 'mysql'
         uses: actions-rs/cargo@v1
         with:
           command: bench
-          args: --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}} sqlx-bench sqlx/${{matrix.backend}} async-std rustorm rustorm/with-${{matrix.backend}} rustorm_dao rust_mysql futures sea-orm sea-orm/sqlx-${{matrix.backend}} criterion/async_std"
+          args: --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}} sqlx-bench sqlx/${{matrix.backend}} tokio rustorm rustorm/with-${{matrix.backend}} rustorm_dao rust_mysql futures sea-orm sea-orm/sqlx-${{matrix.backend}} criterion/async_tokio quaint quaint/mysql quaint/serde-support serde"
 
       - name: Push metrics
         env:
diff --git a/diesel_bench/Cargo.toml b/diesel_bench/Cargo.toml
@@ -12,17 +12,16 @@ autobenches = false
 [dependencies]
 dotenv = "0.15"
 criterion = "0.3.2"
-sqlx = {version = "0.5.0", optional = true}
-async-std = { version = "1.5", optional = true}
+sqlx = {version = "0.5.0", features = ["runtime-tokio-rustls"], optional = true}
+tokio = {version = "1", optional = true}
 rusqlite = {version = "0.25", optional = true}
 rust_postgres = {version = "0.19", optional = true, package = "postgres"}
 rust_mysql = {version = "21.0.1", optional = true, package = "mysql"}
 rustorm = {version = "0.20", optional = true}
 rustorm_dao = {version = "0.20", optional = true}
 quaint = {version = "0.2.0-alpha.13", optional = true}
-tokio = {version = "0.2", optional = true}
 serde = {version = "1", optional = true, features = ["derive"]}
-sea-orm = {version = "0.3", optional = true, features = ["runtime-async-std-rustls"]}
+sea-orm = {version = "0.3", optional = true, features = ["runtime-tokio-rustls"]}
 futures = {version = "0.3", optional = true}
 
 [dependencies.diesel]
@@ -47,7 +46,7 @@ default = []
 postgres = ["diesel/postgres"]
 sqlite = ["diesel/sqlite"]
 mysql = ["diesel/mysql"]
-sqlx-bench = ["sqlx", "async-std", "sqlx/runtime-async-std-rustls"]
+sqlx-bench = ["sqlx", "tokio", "sqlx/runtime-tokio-rustls"]
 
 [patch.crates-io]
-quaint = {git = "https://github.com/prisma/quaint", rev = "ec9384f"}
+quaint = {git = "https://github.com/prisma/quaint", rev = "e077df3"}
diff --git a/diesel_bench/Readme.md b/diesel_bench/Readme.md
@@ -33,9 +33,9 @@ To enable other crates add the following features:
 
 * `SQLx: ` `sqlx-bench sqlx/$backend $backend`
 * `Rustorm`: `rustorm rustorm/with-$backend rustorm_dao $backend`
-* `SeaORM`: `sea-orm sea-orm/sqlx-$backend sqlx async-std criterion/async_std futures $backend`
+* `SeaORM`: `sea-orm sea-orm/sqlx-$backend sqlx tokio criterion/tokio futures $backend`
 * `Quaint`: `quaint quaint/$backend tokio quaint/serde-support serde $backend`
-* `Postgres`: `rust-postgres $backend`
+* `Postgres`: `rust_postgres $backend`
 * `Rusqlite`: `rusqlite $backend`
 * `Mysql`: `rust-mysql $backend`
 
diff --git a/diesel_bench/benches/lib.rs b/diesel_bench/benches/lib.rs
@@ -148,7 +148,7 @@ fn bench_medium_complex_query(c: &mut Criterion) {
             crate::quaint_benches::bench_medium_complex_query(b, *i);
         });
 
-        #[cfg(feature = "sqlx-bench")]
+        #[cfg(all(feature = "sqlx-bench", not(feature = "sqlite")))]
         group.bench_with_input(
             BenchmarkId::new("sqlx_query_as_macro", size),
             size,
diff --git a/diesel_bench/benches/quaint_benches.rs b/diesel_bench/benches/quaint_benches.rs
@@ -58,9 +58,20 @@ fn connect(rt: &mut Runtime) -> Quaint {
 
     let conn = rt.block_on({
         async {
-            let conn = Quaint::new(&db_url).await.unwrap();
+            let conn;
+
+            #[cfg(feature = "sqlite")]
+            {
+                conn = Quaint::new_in_memory().unwrap();
+            }
+
+            #[cfg(not(feature = "sqlite"))]
+            {
+                conn = Quaint::new(&db_url).await.unwrap();
+            }
 
             if cfg!(feature = "sqlite") {
+                #[cfg(feature = "sqlite")]
                 for migration in super::SQLITE_MIGRATION_SQL {
                     conn.execute_raw(migration, &[]).await.unwrap();
                 }
diff --git a/diesel_bench/benches/sea_orm_benches/mod.rs b/diesel_bench/benches/sea_orm_benches/mod.rs
@@ -2,6 +2,7 @@ use criterion::Bencher;
 use sea_orm::entity::*;
 use sea_orm::query::*;
 use sea_orm::DatabaseConnection;
+use tokio::runtime::Runtime;
 
 mod comments;
 mod posts;
@@ -12,14 +13,16 @@ use self::posts::Entity as Post;
 use self::users::Entity as User;
 
 #[cfg(feature = "postgres")]
-fn connection() -> (sqlx::PgPool, DatabaseConnection) {
+fn connection() -> (sqlx::PgPool, DatabaseConnection, Runtime) {
     use sea_orm::SqlxPostgresConnector;
 
+    let rt = Runtime::new().expect("Failed to start runtime");
+
     dotenv::dotenv().ok();
     let connection_url = dotenv::var("PG_DATABASE_URL")
         .or_else(|_| dotenv::var("DATABASE_URL"))
         .expect("DATABASE_URL must be set in order to run tests");
-    async_std::task::block_on(async {
+    let (pool, db) = rt.block_on(async {
         use sqlx::Executor;
         let pool = sqlx::PgPool::connect(&connection_url).await.unwrap();
         pool.execute("TRUNCATE TABLE comments CASCADE;")
@@ -30,19 +33,23 @@ fn connection() -> (sqlx::PgPool, DatabaseConnection) {
 
         let db = SqlxPostgresConnector::from_sqlx_postgres_pool(pool.clone());
         (pool, db)
-    })
+    });
+
+    (pool, db, rt)
 }
 
 #[cfg(feature = "mysql")]
-fn connection() -> (sqlx::MySqlPool, DatabaseConnection) {
-    use sea_orm::SqlxMySqlConnector;
+fn connection() -> (sqlx::MySqlPool, DatabaseConnection, Runtime) {
     use futures::StreamExt;
+    use sea_orm::SqlxMySqlConnector;
+
+    let rt = Runtime::new().expect("Failed to start runtime");
 
     dotenv::dotenv().ok();
     let connection_url = dotenv::var("MYSQL_DATABASE_URL")
         .or_else(|_| dotenv::var("DATABASE_URL"))
         .expect("DATABASE_URL must be set in order to run tests");
-    async_std::task::block_on(async {
+    let (pool, db) = rt.block_on(async {
         use sqlx::Executor;
         let pool = sqlx::MySqlPool::connect(&connection_url).await.unwrap();
         let mut result_stream = pool.execute_many(
@@ -51,23 +58,25 @@ fn connection() -> (sqlx::MySqlPool, DatabaseConnection) {
              TRUNCATE TABLE posts;\
              TRUNCATE TABLE users;\
              SET FOREIGN_KEY_CHECKS = 1;\
-             "
+             ",
         );
         while let Some(e) = result_stream.next().await {
             let _ = e.unwrap();
         }
         let db = SqlxMySqlConnector::from_sqlx_mysql_pool(pool.clone());
         (pool, db)
-    })
+    });
+    (pool, db, rt)
 }
 
 #[cfg(feature = "sqlite")]
-fn connection() -> (sqlx::SqlitePool, DatabaseConnection) {
+fn connection() -> (sqlx::SqlitePool, DatabaseConnection, Runtime) {
     use sea_orm::SqlxSqliteConnector;
 
+    let rt = Runtime::new().expect("Failed to start runtime");
     dotenv::dotenv().ok();
 
-    async_std::task::block_on(async {
+    let (pool, db) = rt.block_on(async {
         use sqlx::Executor;
         let pool = sqlx::SqlitePool::connect("sqlite::memory:").await.unwrap();
 
@@ -81,7 +90,9 @@ fn connection() -> (sqlx::SqlitePool, DatabaseConnection) {
 
         let db = SqlxSqliteConnector::from_sqlx_sqlite_pool(pool.clone());
         (pool, db)
-    })
+    });
+
+    (pool, db, rt)
 }
 
 async fn insert_users(
@@ -101,22 +112,22 @@ async fn insert_users(
 }
 
 pub fn bench_trivial_query(b: &mut Bencher, size: usize) {
-    let (pool, conn) = connection();
+    let (pool, conn, rt) = connection();
 
-    async_std::task::block_on(async {
+    rt.block_on(async {
         insert_users(size, &conn, |_| None).await;
     });
 
-    b.to_async(criterion::async_executor::AsyncStdExecutor)
+    b.to_async(&rt)
         .iter(|| async { User::find().all(&conn).await.unwrap() });
 
-    async_std::task::block_on(async {
+    rt.block_on(async {
         pool.close().await;
     })
 }
 
 pub fn bench_medium_complex_query(b: &mut Bencher, size: usize) {
-    let (pool, conn) = connection();
+    let (pool, conn, rt) = connection();
 
     let hair_color_callback = |i| {
         Some(if i % 2 == 0 {
@@ -125,34 +136,32 @@ pub fn bench_medium_complex_query(b: &mut Bencher, size: usize) {
             String::from("brown")
         })
     };
-    async_std::task::block_on(async {
+    rt.block_on(async {
         insert_users(size, &conn, hair_color_callback).await;
     });
 
-    b.to_async(criterion::async_executor::AsyncStdExecutor)
-        .iter(|| async {
-            let r: Vec<(self::users::Model, Option<self::posts::Model>)> = User::find()
-                .find_also_related(Post)
-                .all(&conn)
-                .await
-                .unwrap();
-            r
-        });
-
-    async_std::task::block_on(async {
+    b.to_async(&rt).iter(|| async {
+        let r: Vec<(self::users::Model, Option<self::posts::Model>)> = User::find()
+            .find_also_related(Post)
+            .all(&conn)
+            .await
+            .unwrap();
+        r
+    });
+
+    rt.block_on(async {
         pool.close().await;
     })
 }
 
 pub fn bench_insert(b: &mut Bencher, size: usize) {
-    let (pool, conn) = connection();
+    let (pool, conn, rt) = connection();
 
-    b.to_async(criterion::async_executor::AsyncStdExecutor)
-        .iter(|| async {
-            insert_users(size, &conn, |_| Some(String::from("hair_color"))).await;
-        });
+    b.to_async(&rt).iter(|| async {
+        insert_users(size, &conn, |_| Some(String::from("hair_color"))).await;
+    });
 
-    async_std::task::block_on(async {
+    rt.block_on(async {
         pool.close().await;
     })
 }
@@ -165,9 +174,9 @@ pub fn loading_associations_sequentially(b: &mut Bencher) {
     const USER_NUMBER: usize = 100;
 
     // SETUP A TON OF DATA
-    let (pool, conn) = connection();
+    let (pool, conn, rt) = connection();
 
-    async_std::task::block_on(async {
+    rt.block_on(async {
         insert_users(USER_NUMBER, &conn, |i| {
             Some(if i % 2 == 0 {
                 "black".to_owned()
@@ -213,54 +222,53 @@ pub fn loading_associations_sequentially(b: &mut Bencher) {
     });
 
     // ACTUAL BENCHMARK
-    b.to_async(criterion::async_executor::AsyncStdExecutor)
-        .iter(|| async {
-            use std::collections::HashMap;
-
-            let res: Vec<(self::users::Model, Vec<self::posts::Model>)> = User::find()
-                .find_with_related(Post)
-                .all(&conn)
-                .await
-                .unwrap();
-
-            let post_ids = res
-                .iter()
-                .flat_map(|(_, posts)| posts.iter().map(|post| post.id))
-                .collect::<Vec<_>>();
-
-            let comments = Comment::find()
-                .filter(self::comments::Column::PostId.is_in(post_ids))
-                .all(&conn)
-                .await
-                .unwrap();
-
-            let mut lookup = HashMap::new();
-
-            for comment in comments {
-                lookup
-                    .entry(comment.post_id)
-                    .or_insert(Vec::new())
-                    .push(comment);
-            }
-
-            res.into_iter()
-                .map(|(user, posts)| {
-                    let posts = posts
-                        .into_iter()
-                        .map(|post| {
-                            let post_id = post.id;
-                            (post, lookup[&post_id].clone())
-                        })
-                        .collect();
-                    (user, posts)
-                })
-                .collect::<Vec<(
-                    self::users::Model,
-                    Vec<(self::posts::Model, Vec<self::comments::Model>)>,
-                )>>()
-        });
+    b.to_async(&rt).iter(|| async {
+        use std::collections::HashMap;
+
+        let res: Vec<(self::users::Model, Vec<self::posts::Model>)> = User::find()
+            .find_with_related(Post)
+            .all(&conn)
+            .await
+            .unwrap();
+
+        let post_ids = res
+            .iter()
+            .flat_map(|(_, posts)| posts.iter().map(|post| post.id))
+            .collect::<Vec<_>>();
+
+        let comments = Comment::find()
+            .filter(self::comments::Column::PostId.is_in(post_ids))
+            .all(&conn)
+            .await
+            .unwrap();
+
+        let mut lookup = HashMap::new();
+
+        for comment in comments {
+            lookup
+                .entry(comment.post_id)
+                .or_insert(Vec::new())
+                .push(comment);
+        }
+
+        res.into_iter()
+            .map(|(user, posts)| {
+                let posts = posts
+                    .into_iter()
+                    .map(|post| {
+                        let post_id = post.id;
+                        (post, lookup[&post_id].clone())
+                    })
+                    .collect();
+                (user, posts)
+            })
+            .collect::<Vec<(
+                self::users::Model,
+                Vec<(self::posts::Model, Vec<self::comments::Model>)>,
+            )>>()
+    });
 
-    async_std::task::block_on(async {
+    rt.block_on(async {
         pool.close().await;
     })
 }
diff --git a/diesel_bench/benches/sqlx_benches.rs b/diesel_bench/benches/sqlx_benches.rs
