Skip to content
Snippets Groups Projects
Commit 1e0124d9 authored by Bruno Burke's avatar Bruno Burke :hamburger:
Browse files

add find-document query caching

parent 436ae249
No related branches found
No related tags found
No related merge requests found
(defproject wwsoftware/document-storage "0.2.4"
(defproject wwsoftware/document-storage "0.2.5"
:description "FIXME: write description"
:repositories [["internal" {:url "https://leukipp.fh-muenster.de:9082/releases"
:username "anonymous"}]]
......
......@@ -53,7 +53,7 @@
#_(def gitstorage (ds/make-git-ds "doc-storage"))
#_(def pgstorage (dspostgres/make-postgres-ds
{:id :mtesting
{:id :main
:repositories {:atoms {:caching true}
:courselists {:caching true}
:courses {:caching true}
......
(ns document-storage.postgres.cache
(:require [document-storage.postgres.encoding :as encoding]
[editscript.core :as escript]
[clj-helper.vector :as vhelper]
[document-storage.query :as dsquery]))
(defn get-cached-document [storage repository document-id version]
(.run-query storage :get-cached-document
{:repository repository
:document-id document-id
:version version}))
(defn cache-document [storage repository id doc]
(swap! (.get-cache storage) update-in [repository :documents] assoc id doc)
doc)
(defn cache-query [storage repository query result]
(swap! (.get-cache storage) update-in [repository :queries] assoc query (into #{} (keys result)))
result)
(defn get-cached-query [storage repository query]
(get-in (.get-cache storage) [repository :queries query]))
(defn uncache-document [storage repository id]
(swap! (.get-cache storage) update-in [repository :queries] dsquery/update-queries-remove-document id)
(swap! (.get-cache storage) update-in [repository :documents] dissoc id))
(defn update-query-cache-document [storage repository document-id document]
(swap! (.get-cache storage) update-in [repository :queries] dsquery/update-queries document-id document))
(defn load-cached-document [storage repository document-id version load-doc-fn]
(if-let [document (get-cached-document storage repository document-id version)]
(encoding/decode-document (:data document))
(let [restored-document (load-doc-fn)]
(.run-query storage :create-document-cache!
{:repository repository
:document-id document-id
:version version
:data (encoding/encode-document restored-document)})
(load-cached-document storage repository document-id version load-doc-fn))))
......@@ -10,6 +10,7 @@
[document-storage.postgres.documents :as documents]
[document-storage.query :refer [query-fits? update-queries
update-queries-remove-document]]
[document-storage.postgres.cache :as dscache]
[conman.core :as conman]
[clj-helper.string :refer [get-unique-id]]
[clj-helper.vector :as vhelper]))
......@@ -23,6 +24,7 @@
(defprotocol db-storage
"Base Protocol for document storages"
(get-schema [this] "Return the db-schema name")
(get-cache [this] "Return the internal cache atom")
(create-repository [this repository] "Create a new repository")
(update-repository [this repository] "Update repository settings")
(delete-repository [this repository] "Delete a repository")
......@@ -36,6 +38,8 @@
db-storage
(get-schema [this]
(.-storage-id this))
(get-cache [this]
(get (.-params this) :cache))
(create-repository [this {:keys [caching versioning] :or {caching false
versioning true}
:as repository}]
......@@ -134,12 +138,18 @@
(find-documents [this query repository]
(let [docs (.list-documents this repository)]
(let [result (into {} (keep (fn [fname]
(let [doc (.load-document this fname repository)]
(when (query-fits? doc query)
[fname doc])))
docs))]
result)))
(or (when-let [cached-query (dscache/get-cached-query this repository query)]
(into {} (map (fn [fname]
(let [doc (.load-document this fname repository)]
[fname doc]))
cached-query)))
(let [result (into {} (keep (fn [fname]
(let [doc (.load-document this fname repository)]
(when (query-fits? doc query)
[fname doc])))
docs))]
(dscache/cache-query this repository query result)))))
(shutdown [this]
(db/disconnect! (:connection this)))
......@@ -155,7 +165,8 @@
(let [conn (db/connect! config)
id (str "ds" (name (or (:id config) "main")))
storage (->postgres-storage id
{:repositories (:repositories config)}
{:repositories (:repositories config)
:cache (atom {})}
(:connection conn)
(:queries conn))]
(.init-storage storage)
......
(ns document-storage.postgres.documents
(:require [document-storage.postgres.encoding :as encoding]
[document-storage.postgres.cache :as cache]
[editscript.core :as escript]
[clj-helper.vector :as vhelper]))
......@@ -48,23 +49,6 @@
:version version
:repository repository}))))))
(defn get-cached-document [storage repository document-id version]
(.run-query storage :get-cached-document
{:repository repository
:document-id document-id
:version version}))
(defn load-cached-document [storage repository document-id version load-doc-fn]
(if-let [document (get-cached-document storage repository document-id version)]
(encoding/decode-document (:data document))
(let [restored-document (load-doc-fn)]
(.run-query storage :create-document-cache!
{:repository repository
:document-id document-id
:version version
:data (encoding/encode-document restored-document)})
(load-cached-document storage repository document-id version load-doc-fn))))
(defn load-document
([storage document-id repository]
(when-let [last-document-version (get-last-document-version storage document-id repository)]
......@@ -72,7 +56,7 @@
([storage document-id version repository]
(let [restore-document-fn #(restore-document-version storage document-id version repository)]
(if (.cached-repository? storage repository)
(load-cached-document storage repository document-id version restore-document-fn)
(cache/load-cached-document storage repository document-id version restore-document-fn)
(restore-document-fn)))))
(defn save-keyframe [storage document-id document last-version repository]
......@@ -104,6 +88,7 @@
:data diff-doc})))
(defn save-document [storage document-id document repository options]
(cache/update-query-cache-document storage repository document-id document)
(if-let [last-version (get-last-document-version storage document-id repository)]
(if (true? (:force-keyframe options))
(save-keyframe storage document-id document last-version repository)
......
......@@ -9,6 +9,7 @@
[document-storage.protocol-test :as protocol-tests]
[document-storage.postgres.core-test :refer [ds-atom storage-fixture]]
[document-storage.postgres.documents :as pgdocs]
[document-storage.postgres.cache :as pgcache]
[document-storage.testdata :refer [generate-document]]
[clj-helper.string :refer [get-unique-id]]))
......@@ -36,22 +37,22 @@
(let [last-version1 (pgdocs/get-last-document-version @ds-atom docname1 repo1name)
last-version2 (pgdocs/get-last-document-version @ds-atom docname2 repo2name)]
;;; Check if inital versions are not cached
(is (nil? (pgdocs/get-cached-document @ds-atom repo1name docname1 last-version1)) "Initial documents are not cached")
(is (nil? (pgdocs/get-cached-document @ds-atom repo2name docname2 last-version2)) "Initial documents are not cached")
(is (nil? (pgcache/get-cached-document @ds-atom repo1name docname1 last-version1)) "Initial documents are not cached")
(is (nil? (pgcache/get-cached-document @ds-atom repo2name docname2 last-version2)) "Initial documents are not cached")
;;; Load doc1 and check if its cached - doc2 should not be cached
(.load-document @ds-atom docname1 repo1name)
(is (not (nil? (pgdocs/get-cached-document @ds-atom repo1name docname1 last-version1))) "Documents should be cached after being loaded at least once")
(is (nil? (pgdocs/get-cached-document @ds-atom repo2name docname2 last-version2)) "Unloaded docs are not cached")
(is (not (nil? (pgcache/get-cached-document @ds-atom repo1name docname1 last-version1))) "Documents should be cached after being loaded at least once")
(is (nil? (pgcache/get-cached-document @ds-atom repo2name docname2 last-version2)) "Unloaded docs are not cached")
;;; Now load doc1 and check if its also cached
(.load-document @ds-atom docname2 repo2name)
(is (not (nil? (pgdocs/get-cached-document @ds-atom repo2name docname2 last-version2))) "Documents should be cached after being loaded at least once")
(is (not (nil? (pgcache/get-cached-document @ds-atom repo2name docname2 last-version2))) "Documents should be cached after being loaded at least once")
;;; Reset repository cache and check if its only reset the chosen repository cache
(.reset-repository-cache @ds-atom repo1name)
(is (nil? (pgdocs/get-cached-document @ds-atom repo1name docname1 last-version1)) "Documents should not be cached after repository-cache reset")
(is (not (nil? (pgdocs/get-cached-document @ds-atom repo2name docname2 last-version2))) "Documents should be cached if another repository is reset")
(is (nil? (pgcache/get-cached-document @ds-atom repo1name docname1 last-version1)) "Documents should not be cached after repository-cache reset")
(is (not (nil? (pgcache/get-cached-document @ds-atom repo2name docname2 last-version2))) "Documents should be cached if another repository is reset")
;;; Reset complete cache and check if it cleared the remaining cache for repo2
(.reset-cache @ds-atom)
(is (nil? (pgdocs/get-cached-document @ds-atom repo2name docname2 last-version2)) "Documents should not be cached if the whole cache was reset")))
(is (nil? (pgcache/get-cached-document @ds-atom repo2name docname2 last-version2)) "Documents should not be cached if the whole cache was reset")))
(.delete-repository @ds-atom repo1name)
(.delete-repository @ds-atom repo2name))))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment