Separate fetching a single identity and fetching many in DataSource

docs/src/jekyll/_layouts/docs.html

@@ -1,5 +1 @@
-{% include_remote https://rawgit.com/47deg/microsites/cdn/docs.html %}
-
-<!-- https://cdn.rawgit.com/47deg/microsites/cdn/docs.html -->
-<!-- https://rawgit.com/47deg/microsites/cdn/docs.html -->
-<!-- http://localhost/~rafaparadela/cdn47/microsites/docs.html -->
+{% include_remote https://cdn.rawgit.com/47deg/microsites/cdn/docs.html %}

docs/src/jekyll/_layouts/home.html

@@ -1 +1 @@
-{% include_remote  https://rawgit.com/47deg/microsites/cdn/home.html %}
+{% include_remote https://cdn.rawgit.com/47deg/microsites/cdn/home.html %}

docs/src/tut/docs.md

@@ -65,7 +65,8 @@ In order to tell Fetch how to retrieve data, we must implement the `DataSource`
 
 ```scala
 trait DataSource[Identity, Result]{
-  def fetch(ids: NonEmptyList[Identity]): Eval[Map[Identity, Result]]
+  def fetchOne(id: Identity): Eval[Option[Result]]
+  def fetchMany(ids: NonEmptyList[Identity]): Eval[Map[Identity, Result]]
 }
 ```
 
@@ -74,8 +75,12 @@ It takes two type parameters:
  - `Identity`: the identity we want to fetch (a `UserId` if we were fetching users)
  - `Result`: the type of the data we retrieve (a `User` if we were fetching users)
 
-The `fetch` method takes a non-empty list of identities and must return an [Eval](https://github.com/typelevel/cats/blob/master/core/src/main/scala/cats/Eval.scala) that will result
-in a map from identities to results. Accepting a list of identities gives Fetch the ability to batch requests to
+There are two methods: `fetchOne` and `fetchMany`. `fetchOne` receives one identity and must return
+an [Eval](https://github.com/typelevel/cats/blob/master/core/src/main/scala/cats/Eval.scala) containing
+an optional result. Returning an `Option` Fetch can detect whether an identity couldn't be fetched or no longer exists.
+
+`fetchMany` method takes a non-empty list of identities and must return an `Eval`  that containing
+a map from identities to results. Accepting a list of identities gives Fetch the ability to batch requests to
 the same data source, and returning a mapping from identities to results, Fetch can detect whenever an identity
 couldn't be fetched or no longer exists.
 
@@ -107,7 +112,13 @@ val userDatabase: Map[UserId, User] = Map(
 )
 
 implicit object UserSource extends DataSource[UserId, User]{
-  override def fetch(ids: NonEmptyList[UserId]): Eval[Map[UserId, User]] = {
+  override def fetchOne(id: UserId): Eval[Option[User]] = {
+    Eval.later({
+      println(s"Fetching user $id")
+      userDatabase.get(id)
+    })
+  }
+  override def fetchMany(ids: NonEmptyList[UserId]): Eval[Map[UserId, User]] = {
     Eval.later({
       println(s"Fetching users $ids")
       userDatabase.filterKeys(ids.unwrap.contains)
@@ -123,6 +134,23 @@ given an id, we just have to pass a `UserId` as an argument to `Fetch`.
 def getUser(id: UserId): Fetch[User] = Fetch(id) // or, more explicitly: Fetch(id)(UserSource)
 ```
 
+
+### Data sources that don't support batching
+
+If your data source doesn't support batching, you can use the `DataSource#batchingNotSupported` method as the implementation
+of `fetchMany`. Note that it will use the `fetchOne` implementation for requesting identities one at a time.
+
+```tut:silent
+implicit object IntSource extends DataSource[Int, Int]{
+  override def fetchOne(id: Int): Eval[Option[Int]] = {
+    Eval.now(Option(id))
+  }
+  override def fetchMany(ids: NonEmptyList[Int]): Eval[Map[Int, Int]] = {
+    batchingNotSupported(ids)
+  }
+}
+```
+
 ## Creating and running a fetch
 
 We are now ready to create and run fetches. Note the distinction between Fetch creation and execution.
@@ -258,7 +286,13 @@ val postDatabase: Map[PostId, Post] = Map(
 )
 
 implicit object PostSource extends DataSource[PostId, Post]{
-  override def fetch(ids: NonEmptyList[PostId]): Eval[Map[PostId, Post]] = {
+  override def fetchOne(id: PostId): Eval[Option[Post]] = {
+    Eval.later({
+      println(s"Fetching post $id")
+      postDatabase.get(id)
+    })
+  }
+  override def fetchMany(ids: NonEmptyList[PostId]): Eval[Map[PostId, Post]] = {
     Eval.later({
       println(s"Fetching posts $ids")
       postDatabase.filterKeys(ids.unwrap.contains)
@@ -275,7 +309,13 @@ val postInfoDatabase: Map[PostId, PostInfo] = Map(
 )
 
 implicit object PostInfoSource extends DataSource[PostId, PostInfo]{
-  override def fetch(ids: NonEmptyList[PostId]): Eval[Map[PostId, PostInfo]] = {
+  override def fetchOne(id: PostId): Eval[Option[PostInfo]] = {
+    Eval.later({
+      println(s"Fetching post info $id")
+      postInfoDatabase.get(id)
+    })
+  }
+  override def fetchMany(ids: NonEmptyList[PostId]): Eval[Map[PostId, PostInfo]] = {
     Eval.later({
       println(s"Fetching post info $ids")
       postInfoDatabase.filterKeys(ids.unwrap.contains)

docs/src/tut/index.md

@@ -41,7 +41,8 @@ Data Sources take two type parameters:
 
 ```scala
 trait DataSource[Identity, Result]{
-  def fetch(ids: NonEmptyList[Identity]): Eval[Map[Identity, Result]]
+  def fetchOne(id: Identity): Eval[Option[Result]]
+  def fetchMany(ids: NonEmptyList[Identity]): Eval[Map[Identity, Result]]
 }
 ```
 
@@ -55,7 +56,13 @@ import cats.std.list._
 import fetch._
 
 implicit object ToStringSource extends DataSource[Int, String]{
-  override def fetch(ids: NonEmptyList[Int]): Eval[Map[Int, String]] = {
+  override def fetchOne(id: Int): Eval[Option[String]] = {
+    Eval.later({
+      println(s"ToStringSource $id")
+      Option(id.toString)
+    })
+  }
+  override def fetchMany(ids: NonEmptyList[Int]): Eval[Map[Int, String]] = {
     Eval.later({
       println(s"ToStringSource $ids")
       ids.unwrap.map(i => (i, i.toString)).toMap
@@ -107,7 +114,13 @@ If we combine two independent fetches from different data sources, the fetches w
 
 ```tut:silent
 implicit object LengthSource extends DataSource[String, Int]{
-  override def fetch(ids: NonEmptyList[String]): Eval[Map[String, Int]] = {
+  override def fetchOne(id: String): Eval[Option[Int]] = {
+    Eval.later({
+      println(s"LengthSource $id")
+      Option(id.size)
+    })
+  }
+  override def fetchMany(ids: NonEmptyList[String]): Eval[Map[String, Int]] = {
     Eval.later({
       println(s"LengthSource $ids")
       ids.unwrap.map(i => (i, i.size)).toMap

shared/src/main/scala/datasource.scala

@@ -19,12 +19,48 @@ package fetch
 import cats.Eval
 import cats.data.NonEmptyList
 
+import cats.std.list._
+import cats.syntax.traverse._
+
 /**
   * A `DataSource` is the recipe for fetching a certain identity `I`, which yields
   * results of type `A`.
   */
 trait DataSource[I, A] {
-  def name: DataSourceName               = this.toString
+
+  /** The name of the data source.
+    */
+  def name: DataSourceName = this.toString
+
+  /**
+    * Derive a `DataSourceIdentity` from an identity, suitable for storing the result
+    * of such identity in a `DataSourceCache`.
+    */
   def identity(i: I): DataSourceIdentity = (name, i)
-  def fetch(ids: NonEmptyList[I]): Eval[Map[I, A]]
+
+  /** Fetch one identity, returning a None if it wasn't found.
+    */
+  def fetchOne(id: I): Eval[Option[A]]
+
+  /** Fetch many identities, returning a mapping from identities to results. If an
+    * identity wasn't found won't appear in the keys.
+    */
+  def fetchMany(ids: NonEmptyList[I]): Eval[Map[I, A]]
+
+  /** Use `fetchOne` for implementing of `fetchMany`. Use only when the data
+    * source doesn't support batching.
+    */
+  def batchingNotSupported(ids: NonEmptyList[I]): Eval[Map[I, A]] = {
+    val idsList = ids.unwrap
+    idsList
+      .map(fetchOne)
+      .sequence
+      .map(results => {
+        (idsList zip results)
+          .collect({
+            case (id, Some(result)) => (id, result)
+          })
+          .toMap
+      })
+  }
 }

shared/src/main/scala/interpreters.scala

@@ -75,7 +75,7 @@ trait FetchInterpreters {
                       case (ds, as) =>
                         MM.pureEval(ds
                               .asInstanceOf[DataSource[I, A]]
-                              .fetch(as.asInstanceOf[NonEmptyList[I]]))
+                              .fetchMany(as.asInstanceOf[NonEmptyList[I]]))
                     })
                         .sequence)((results: List[Map[_, _]]) => {
                     val endRound = System.nanoTime()
@@ -121,42 +121,39 @@ trait FetchInterpreters {
                 cache
                   .get(ds.identity(id))
                   .fold[M[(FetchEnv, A)]](
-                      MM.flatMap(
-                          MM.pureEval(ds.fetch(NonEmptyList(id))).asInstanceOf[M[Map[I, A]]])(
-                          (res: Map[I, A]) => {
+                      MM.flatMap(MM.pureEval(ds.fetchOne(id)).asInstanceOf[M[Option[A]]])(
+                          (res: Option[A]) => {
                         val endRound = System.nanoTime()
-                        res
-                          .get(id.asInstanceOf[I])
-                          .fold[M[(FetchEnv, A)]](
-                              MM.raiseError(
-                                  FetchFailure(
-                                      env.next(
-                                          cache,
-                                          Round(cache,
-                                                ds.name,
-                                                OneRound(id),
-                                                startRound,
-                                                endRound),
-                                          List(id)
-                                      )
-                                  )
-                              )
-                          )(result => {
-                            val endRound = System.nanoTime()
-                            val newCache = cache.update(ds.identity(id), result)
-                            MM.pure(
-                                (env.next(
-                                     newCache,
-                                     Round(cache,
-                                           ds.name,
-                                           OneRound(id),
-                                           startRound,
-                                           endRound),
-                                     List(id)
-                                 ),
-                                 result)
+                        res.fold[M[(FetchEnv, A)]](
+                            MM.raiseError(
+                                FetchFailure(
+                                    env.next(
+                                        cache,
+                                        Round(cache,
+                                              ds.name,
+                                              OneRound(id),
+                                              startRound,
+                                              endRound),
+                                        List(id)
+                                    )
+                                )
                             )
-                          })
+                        )(result => {
+                          val endRound = System.nanoTime()
+                          val newCache = cache.update(ds.identity(id), result)
+                          MM.pure(
+                              (env.next(
+                                   newCache,
+                                   Round(cache,
+                                         ds.name,
+                                         OneRound(id),
+                                         startRound,
+                                         endRound),
+                                   List(id)
+                               ),
+                               result)
+                          )
+                        })
                       })
                   )(cached => {
                     val endRound = System.nanoTime()
@@ -196,7 +193,7 @@ trait FetchInterpreters {
                   )
                 else {
                   MM.flatMap(MM
-                        .pureEval(ds.fetch(NonEmptyList(newIds(0), newIds.tail)))
+                        .pureEval(ds.fetchMany(NonEmptyList(newIds(0), newIds.tail)))
                         .asInstanceOf[M[Map[I, A]]])((res: Map[I, A]) => {
                     val endRound = System.nanoTime()
                     ids.unwrap

shared/src/test/scala/FetchTests.scala

@@ -29,36 +29,44 @@ object TestHelper {
 
   val M: MonadError[Eval, Throwable] = implicits.evalMonadError
 
-  case class NotFound() extends Throwable
+  final case class NotFound() extends Throwable
 
-  case class One(id: Int)
+  final case class One(id: Int)
   implicit object OneSource extends DataSource[One, Int] {
     override def name = "OneSource"
-    override def fetch(ids: NonEmptyList[One]): Eval[Map[One, Int]] =
+    override def fetchOne(id: One): Eval[Option[Int]] = {
+      M.pure(Option(id.id))
+    }
+    override def fetchMany(ids: NonEmptyList[One]): Eval[Map[One, Int]] =
       M.pure(ids.unwrap.map(one => (one, one.id)).toMap)
   }
   def one(id: Int): Fetch[Int] = Fetch(One(id))
 
-  case class AnotherOne(id: Int)
+  final case class AnotherOne(id: Int)
   implicit object AnotheroneSource extends DataSource[AnotherOne, Int] {
     override def name = "AnotherOneSource"
-
-    override def fetch(ids: NonEmptyList[AnotherOne]): Eval[Map[AnotherOne, Int]] =
+    override def fetchOne(id: AnotherOne): Eval[Option[Int]] =
+      M.pure(Option(id.id))
+    override def fetchMany(ids: NonEmptyList[AnotherOne]): Eval[Map[AnotherOne, Int]] =
       M.pure(ids.unwrap.map(anotherone => (anotherone, anotherone.id)).toMap)
   }
   def anotherOne(id: Int): Fetch[Int] = Fetch(AnotherOne(id))
 
-  case class Many(n: Int)
+  final case class Many(n: Int)
   implicit object ManySource extends DataSource[Many, List[Int]] {
     override def name = "ManySource"
-    override def fetch(ids: NonEmptyList[Many]): Eval[Map[Many, List[Int]]] =
+    override def fetchOne(id: Many): Eval[Option[List[Int]]] =
+      M.pure(Option(0 until id.n toList))
+    override def fetchMany(ids: NonEmptyList[Many]): Eval[Map[Many, List[Int]]] =
       M.pure(ids.unwrap.map(m => (m, 0 until m.n toList)).toMap)
   }
 
-  case class Never()
+  final case class Never()
   implicit object NeverSource extends DataSource[Never, Int] {
     override def name = "NeverSource"
-    override def fetch(ids: NonEmptyList[Never]): Eval[Map[Never, Int]] =
+    override def fetchOne(id: Never): Eval[Option[Int]] =
+      M.pure(None)
+    override def fetchMany(ids: NonEmptyList[Never]): Eval[Map[Never, Int]] =
       M.pure(Map.empty[Never, Int])
   }
   def many(id: Int): Fetch[List[Int]] = Fetch(Many(id))
@@ -595,7 +603,7 @@ class FetchTests extends FreeSpec with Matchers {
     totalFetched(rounds) shouldEqual 0
   }
 
-  case class MyCache(state: Map[Any, Any] = Map.empty[Any, Any]) extends DataSourceCache {
+  final case class MyCache(state: Map[Any, Any] = Map.empty[Any, Any]) extends DataSourceCache {
     override def get(k: DataSourceIdentity): Option[Any] = state.get(k)
     override def update[A](k: DataSourceIdentity, v: A): MyCache =
       copy(state = state.updated(k, v))
@@ -649,14 +657,16 @@ class FetchFutureTests extends AsyncFreeSpec with Matchers {
   implicit def executionContext = global
   override def newInstance      = new FetchFutureTests
 
-  case class ArticleId(id: Int)
-  case class Article(id: Int, content: String) {
+  final case class ArticleId(id: Int)
+  final case class Article(id: Int, content: String) {
     def author: Int = id + 1
   }
 
   implicit object ArticleFuture extends DataSource[ArticleId, Article] {
     override def name = "ArticleFuture"
-    override def fetch(ids: NonEmptyList[ArticleId]): Eval[Map[ArticleId, Article]] = {
+    override def fetchOne(id: ArticleId): Eval[Option[Article]] =
+      Eval.later(Option(Article(id.id, "An article with id " + id.id)))
+    override def fetchMany(ids: NonEmptyList[ArticleId]): Eval[Map[ArticleId, Article]] = {
       Eval.later({
         ids.unwrap.map(tid => (tid, Article(tid.id, "An article with id " + tid.id))).toMap
       })
@@ -665,12 +675,14 @@ class FetchFutureTests extends AsyncFreeSpec with Matchers {
 
   def article(id: Int): Fetch[Article] = Fetch(ArticleId(id))
 
-  case class AuthorId(id: Int)
-  case class Author(id: Int, name: String)
+  final case class AuthorId(id: Int)
+  final case class Author(id: Int, name: String)
 
   implicit object AuthorFuture extends DataSource[AuthorId, Author] {
     override def name = "AuthorFuture"
-    override def fetch(ids: NonEmptyList[AuthorId]): Eval[Map[AuthorId, Author]] = {
+    override def fetchOne(id: AuthorId): Eval[Option[Author]] =
+      Eval.later(Option(Author(id.id, "@egg" + id.id)))
+    override def fetchMany(ids: NonEmptyList[AuthorId]): Eval[Map[AuthorId, Author]] = {
       Eval.later({
         ids.unwrap.map(tid => (tid, Author(tid.id, "@egg" + tid.id))).toMap
       })