add find-document query caching

project.clj

-(defproject wwsoftware/document-storage "0.2.4"
+(defproject wwsoftware/document-storage "0.2.5"
   :description "FIXME: write description"
   :repositories [["internal" {:url "https://leukipp.fh-muenster.de:9082/releases"
                               :username "anonymous"}]]

src/document_storage/migrate.clj

@@ -53,7 +53,7 @@
 
 #_(def gitstorage (ds/make-git-ds "doc-storage"))
 #_(def pgstorage (dspostgres/make-postgres-ds
-                  {:id :mtesting
+                  {:id :main
                    :repositories {:atoms {:caching true}
                                   :courselists {:caching true}
                                   :courses {:caching true}

src/document_storage/postgres/cache.clj

+(ns document-storage.postgres.cache
+  (:require [document-storage.postgres.encoding :as encoding]
+            [editscript.core :as escript]
+            [clj-helper.vector :as vhelper]
+            [document-storage.query :as dsquery]))
+
+(defn get-cached-document [storage repository document-id version]
+  (.run-query storage :get-cached-document
+              {:repository repository
+               :document-id document-id
+               :version version}))
+
+(defn cache-document [storage repository id doc]
+  (swap! (.get-cache storage) update-in [repository :documents] assoc id doc)
+  doc)
+
+(defn cache-query [storage repository query result]
+  (swap! (.get-cache storage) update-in [repository :queries] assoc query (into #{} (keys result)))
+  result)
+
+
+(defn get-cached-query [storage repository query]
+  (get-in (.get-cache storage) [repository :queries query]))
+
+(defn uncache-document [storage repository id]
+  (swap! (.get-cache storage) update-in [repository :queries] dsquery/update-queries-remove-document id)
+  (swap! (.get-cache storage) update-in [repository :documents] dissoc id))
+
+(defn update-query-cache-document [storage repository document-id document]
+  (swap! (.get-cache storage) update-in [repository :queries] dsquery/update-queries document-id document))
+
+
+(defn load-cached-document [storage repository document-id version load-doc-fn]
+  (if-let [document (get-cached-document storage repository document-id version)]
+    (encoding/decode-document (:data document))
+    (let [restored-document (load-doc-fn)]
+      (.run-query storage :create-document-cache!
+                  {:repository repository
+                   :document-id document-id
+                   :version version
+                   :data (encoding/encode-document restored-document)})
+      (load-cached-document storage repository document-id version load-doc-fn))))

src/document_storage/postgres/core.clj

@@ -10,6 +10,7 @@
             [document-storage.postgres.documents :as documents]
             [document-storage.query :refer [query-fits? update-queries
                                             update-queries-remove-document]]
+            [document-storage.postgres.cache :as dscache]
             [conman.core :as conman]
             [clj-helper.string :refer [get-unique-id]]
             [clj-helper.vector :as vhelper]))
@@ -23,6 +24,7 @@
 (defprotocol db-storage
   "Base Protocol for document storages"
   (get-schema [this] "Return the db-schema name")
+  (get-cache [this] "Return the internal cache atom")
   (create-repository [this repository] "Create a new repository")
   (update-repository [this repository] "Update repository settings")
   (delete-repository [this repository] "Delete a repository")
@@ -36,6 +38,8 @@
   db-storage
   (get-schema [this]
     (.-storage-id this))
+  (get-cache [this]
+    (get (.-params this) :cache))
   (create-repository [this {:keys [caching versioning] :or {caching false
                                                             versioning true}
                             :as repository}]
@@ -134,12 +138,18 @@
 
   (find-documents [this query repository]
     (let [docs (.list-documents this repository)]
-      (let [result (into {} (keep (fn [fname]
-                                    (let [doc (.load-document this fname repository)]
-                                      (when (query-fits? doc query)
-                                        [fname doc])))
-                                  docs))]
-        result)))
+      (or (when-let [cached-query (dscache/get-cached-query this repository query)]
+              (into {} (map (fn [fname]
+                              (let [doc (.load-document this fname repository)]
+                                [fname doc]))
+                            cached-query)))
+          (let [result (into {} (keep (fn [fname]
+                                        (let [doc (.load-document this fname repository)]
+                                          (when (query-fits? doc query)
+                                            [fname doc])))
+
+                                      docs))]
+              (dscache/cache-query this repository query result)))))
 
   (shutdown [this]
     (db/disconnect! (:connection this)))
@@ -155,7 +165,8 @@
   (let [conn (db/connect! config)
         id (str "ds" (name (or (:id config) "main")))
         storage (->postgres-storage id
-                                    {:repositories (:repositories config)}
+                                    {:repositories (:repositories config)
+                                     :cache (atom {})}
                                     (:connection conn)
                                     (:queries conn))]
     (.init-storage storage)

src/document_storage/postgres/documents.clj

 (ns document-storage.postgres.documents
   (:require [document-storage.postgres.encoding :as encoding]
+            [document-storage.postgres.cache :as cache]
             [editscript.core :as escript]
             [clj-helper.vector :as vhelper]))
 
@@ -48,23 +49,6 @@
                    :version version
                    :repository repository}))))))
 
-(defn get-cached-document [storage repository document-id version]
-  (.run-query storage :get-cached-document
-              {:repository repository
-               :document-id document-id
-               :version version}))
-
-(defn load-cached-document [storage repository document-id version load-doc-fn]
-  (if-let [document (get-cached-document storage repository document-id version)]
-    (encoding/decode-document (:data document))
-    (let [restored-document (load-doc-fn)]
-      (.run-query storage :create-document-cache!
-                  {:repository repository
-                   :document-id document-id
-                   :version version
-                   :data (encoding/encode-document restored-document)})
-      (load-cached-document storage repository document-id version load-doc-fn))))
-
 (defn load-document
   ([storage document-id repository]
    (when-let [last-document-version (get-last-document-version storage document-id repository)]
@@ -72,7 +56,7 @@
   ([storage document-id version repository]
    (let [restore-document-fn #(restore-document-version storage document-id version repository)]
      (if (.cached-repository? storage repository)
-       (load-cached-document storage repository document-id version restore-document-fn)
+       (cache/load-cached-document storage repository document-id version restore-document-fn)
        (restore-document-fn)))))
 
 (defn save-keyframe [storage document-id document last-version repository]
@@ -104,6 +88,7 @@
                  :data diff-doc})))
 
 (defn save-document [storage document-id document repository options]
+  (cache/update-query-cache-document storage repository document-id document)
   (if-let [last-version (get-last-document-version storage document-id repository)]
     (if (true? (:force-keyframe options))
       (save-keyframe storage document-id document last-version repository)

test/document_storage/postgres/caching_test.clj

@@ -9,6 +9,7 @@
             [document-storage.protocol-test :as protocol-tests]
             [document-storage.postgres.core-test :refer [ds-atom storage-fixture]]
             [document-storage.postgres.documents :as pgdocs]
+            [document-storage.postgres.cache :as pgcache]
             [document-storage.testdata :refer [generate-document]]
             [clj-helper.string :refer [get-unique-id]]))
 
@@ -36,22 +37,22 @@
         (let [last-version1 (pgdocs/get-last-document-version @ds-atom docname1 repo1name)
               last-version2 (pgdocs/get-last-document-version @ds-atom docname2 repo2name)]
           ;;; Check if inital versions are not cached
-          (is (nil? (pgdocs/get-cached-document @ds-atom repo1name docname1 last-version1)) "Initial documents are not cached")
-          (is (nil? (pgdocs/get-cached-document @ds-atom repo2name docname2 last-version2)) "Initial documents are not cached")
+          (is (nil? (pgcache/get-cached-document @ds-atom repo1name docname1 last-version1)) "Initial documents are not cached")
+          (is (nil? (pgcache/get-cached-document @ds-atom repo2name docname2 last-version2)) "Initial documents are not cached")
           ;;; Load doc1 and check if its cached - doc2 should not be cached
           (.load-document @ds-atom docname1 repo1name)
-          (is (not (nil? (pgdocs/get-cached-document @ds-atom repo1name docname1 last-version1))) "Documents should be cached after being loaded at least once")
-          (is (nil? (pgdocs/get-cached-document @ds-atom repo2name docname2 last-version2)) "Unloaded docs are not cached")
+          (is (not (nil? (pgcache/get-cached-document @ds-atom repo1name docname1 last-version1))) "Documents should be cached after being loaded at least once")
+          (is (nil? (pgcache/get-cached-document @ds-atom repo2name docname2 last-version2)) "Unloaded docs are not cached")
           ;;; Now load doc1 and check if its also cached
           (.load-document @ds-atom docname2 repo2name)
-          (is (not (nil? (pgdocs/get-cached-document @ds-atom repo2name docname2 last-version2))) "Documents should be cached after being loaded at least once")
+          (is (not (nil? (pgcache/get-cached-document @ds-atom repo2name docname2 last-version2))) "Documents should be cached after being loaded at least once")
 
           ;;; Reset repository cache and check if its only reset the chosen repository cache
           (.reset-repository-cache @ds-atom repo1name)
-          (is (nil? (pgdocs/get-cached-document @ds-atom repo1name docname1 last-version1)) "Documents should not be cached after repository-cache reset")
-          (is (not (nil? (pgdocs/get-cached-document @ds-atom repo2name docname2 last-version2))) "Documents should be cached if another repository is reset")
+          (is (nil? (pgcache/get-cached-document @ds-atom repo1name docname1 last-version1)) "Documents should not be cached after repository-cache reset")
+          (is (not (nil? (pgcache/get-cached-document @ds-atom repo2name docname2 last-version2))) "Documents should be cached if another repository is reset")
           ;;; Reset complete cache and check if it cleared the remaining cache for repo2
           (.reset-cache @ds-atom)
-          (is (nil? (pgdocs/get-cached-document @ds-atom repo2name docname2 last-version2)) "Documents should not be cached if the whole cache was reset")))
+          (is (nil? (pgcache/get-cached-document @ds-atom repo2name docname2 last-version2)) "Documents should not be cached if the whole cache was reset")))
       (.delete-repository @ds-atom repo1name)
       (.delete-repository @ds-atom repo2name))))