add caching and versioning parameters; add generated column datasize for document fragments

resources/sql/_ds_postgres.sql

@@ -12,6 +12,10 @@ SELECT * FROM dstorage.users
 -- :doc retrieve a list of all repositories.
 SELECT * FROM :i:repositories-table
 
+-- :name get-repository-info :? :1
+-- :doc retrieve repository infos.
+SELECT * FROM :i:repositories-table WHERE id = :repository-name
+
 
 -- :name get-document-frame :? :1
 -- :doc retrieve a document-frame given the uuid.

resources/sql/_ds_postgres_repository.sql

@@ -9,6 +9,7 @@ CREATE TABLE :i:repository-table (
 	reference varchar(32) NULL,
 	created_at timestamptz(0) NOT NULL DEFAULT CURRENT_TIMESTAMP,
 	"data" bytea NOT NULL,
+  datasize numeric NULL GENERATED ALWAYS AS (length("data")) STORED,
 	"version" int8 NOT NULL DEFAULT 0,
 	data_encoding :i:ds-encoding-fqn NOT NULL,
 	CONSTRAINT
@@ -24,8 +25,8 @@ CREATE INDEX
   ON :i:repository-table USING btree (document_id);
 --;;
 INSERT INTO :i:repositories-table
-            (id)
-VALUES (:repository-name)
+            (id, caching, versioning)
+VALUES (:repository-name, :caching, :versioning)
 
 
 -- :name delete-repository!

resources/sql/_ds_postgres_storage.sql

@@ -6,6 +6,8 @@ CREATE SCHEMA IF NOT EXISTS :i:schema-name;
 --;;
 CREATE TABLE IF NOT EXISTS :i:repositories-table (
 	id varchar(32) NOT NULL,
+  caching bool NOT NULL DEFAULT FALSE,
+	"versioning" bool NOT NULL DEFAULT TRUE,
 	created_at timestamptz(0) NOT NULL DEFAULT CURRENT_TIMESTAMP,
 	CONSTRAINT repositories_pk PRIMARY KEY (id));
 --;;

src/document_storage/postgres/core.clj

@@ -11,28 +11,35 @@
             [document-storage.query :refer [query-fits? update-queries
                                             update-queries-remove-document]]
             [conman.core :as conman]
-            [clj-helper.string :refer [get-unique-id]]))
+            [clj-helper.string :refer [get-unique-id]]
+            [clj-helper.vector :as vhelper]))
 
 (defn prepare-repository-name [repo-id]
   (-> (if (keyword? repo-id)
         (name repo-id)
         (str repo-id))
-      (str/replace "-" "_")
-      ))
+      (str/replace "-" "_")))
 
 (defprotocol db-storage
   "Base Protocol for document storages"
   (get-schema [this] "Return the db-schema name")
   (create-repository [this repository] "Create a new repository")
   (delete-repository [this repository] "Delete a repository")
+  (cached-repository? [this repository] "Checks if the repository will use caching")
   (run-query [this query-key params] "Run a Query on a DB-based Storage"))
 
 (defrecord postgres-storage [storage-id params connection queries]
   db-storage
   (get-schema [this]
     (.-storage-id this))
-  (create-repository [this repository]
-    (.run-query this :create-repository! {:repository repository}))
+  (create-repository [this {:keys [caching versioning] :or {caching false
+                                                            versioning true}
+                            :as repository}]
+    (log/info repository)
+    (.run-query this :create-repository!
+                {:repository (:name repository)
+                 :caching caching
+                 :versioning versioning}))
   (delete-repository [this repository]
     (.run-query this :delete-repository! {:repository repository}))
   (run-query [this query-key params]
@@ -57,18 +64,25 @@
                        :uuid (get-unique-id "ds")}]
       (conman/query (:queries this) query-key
                     (merge table-names params))))
+  (cached-repository? [this repository]
+    (let [repo-info (run-query this :get-repository-info
+                               {:repository repository})]
+      (:caching repo-info)))
 
   protocol/storage
   (init-storage [this]
     (log/info "Init Postgres-Storage.")
     (.run-query this :create-storage! {})
-    (let [repositories (set (map prepare-repository-name (get (.-params this) :repositories [])))
+    (let [repositories (map (fn [[id repo-params]]
+                              (assoc repo-params :name (prepare-repository-name id)))
+                            (get (.-params this) :repositories {}))
+          repository-names (set (map :name repositories))
           existing-repos (set (map :id (.list-repositories this)))]
-      (log/info "Repositories: " repositories)
+      (log/info "Repositories: " repository-names)
       (log/info "Existing Repositories: " existing-repos)
-      (doseq [r (difference repositories existing-repos)]
+      (doseq [r (difference repository-names existing-repos)]
         (log/info "Create Repository: " r)
-        (.create-repository this r))))
+        (.create-repository this (vhelper/get-by repositories :name r)))))
 
   (list-repositories [this]
     (run-query this :get-repositories {}))
@@ -94,8 +108,9 @@
                                        {:repository repository}))))
 
   (versioned-repository? [this repository]
-    ;;;TODO check if repo is versioned
-    true)
+    (let [repo-info (run-query this :get-repository-info
+                               {:repository repository})]
+      (:versioning repo-info)))
 
   (find-documents [this query repository]
     (let [docs (.list-documents this repository)]
@@ -103,7 +118,6 @@
                                     (let [doc (.load-document this fname repository)]
                                       (when (query-fits? doc query)
                                         [fname doc])))
-
                                   docs))]
         result)))
 
@@ -120,6 +134,9 @@
 (defn make-postgres-ds [config]
   (let [conn (db/connect! config)
         id (str "ds" (name (or (:id config) "main")))
-        storage (->postgres-storage id {:repositories (:repositories config)} (:connection conn) (:queries conn))]
+        storage (->postgres-storage id
+                                    {:repositories (:repositories config)}
+                                    (:connection conn)
+                                    (:queries conn))]
     (.init-storage storage)
     storage))

src/document_storage/postgres/documents.clj

@@ -16,6 +16,7 @@
                :id frame-id}))
 
 (defn get-last-document-frame [storage document-id repository]
+  ;;;TODO check if this can be speed up by using index in db
   (when-let [frame (.run-query storage :get-last-document-frame
                              {:repository repository
                               :document-id document-id})]
@@ -92,11 +93,11 @@
 
 
 (defn delete-document [storage document-id repository]
-    (.run-query storage :create-document-frame!
-                {:repository repository
-                 :document-id document-id
-                 :type :ds-type/delete
-                 :encoding :ds-encoding/edn
-                 :version (inc (get-last-document-version storage document-id repository))
-                 :reference nil
-                 :data ""}))
+  (.run-query storage :create-document-frame!
+              {:repository repository
+               :document-id document-id
+               :type :ds-type/delete
+               :encoding :ds-encoding/edn
+               :version (inc (get-last-document-version storage document-id repository))
+               :reference nil
+               :data (encoding/encode-document "")}))

test/document_storage/postgres/core_test.clj

@@ -14,7 +14,7 @@
   (reset! ds-atom
           (dspostgres/make-postgres-ds
            {:id :testing
-            :repositories [:testrepo]
+            :repositories {:testrepo {}}
             :jdbc-url (env :database-url)}))
   (f)
   (.delete-repository @ds-atom :testrepo))
@@ -30,3 +30,6 @@
 
 (deftest find-document-test
   (protocol-tests/test-find-documents ds-atom))
+
+
+;;;TODO test if versioning/caching parameters are set correctly for new repositories