postgres: save documents as escript-diffs and keyframes, use nippy for serialization

project.clj

@@ -8,8 +8,10 @@
   :dependencies [[org.clojure/clojure "1.10.1"]
                  [org.clojure/tools.logging "1.1.0"]
                  [com.fzakaria/slf4j-timbre "0.3.20"]
+                 [com.taoensso/nippy "3.1.1"]
                  [clj-jgit "1.0.0-beta3"]
                  [fipp "0.6.21"]
+                 [juji/editscript "0.5.7"]
                  [luminus-migrations "0.7.1"]
                  [com.cognitect/transit-clj "1.0.324"]
                  [conman "0.9.0"]

resources/sql/_ds_postgres_repository.sql

@@ -8,7 +8,7 @@ CREATE TABLE :i:repository-table (
 	data_type :i:ds-type-fqn NOT NULL,
 	reference varchar(32) NULL,
 	created_at timestamptz(0) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-	"data" text NOT NULL,
+	"data" bytea NOT NULL,
 	"version" int8 NOT NULL DEFAULT 0,
 	data_encoding :i:ds-encoding-fqn NOT NULL,
 	CONSTRAINT

resources/sql/_ds_postgres_storage.sql

@@ -29,7 +29,8 @@ DO $$DECLARE
       :i:ds-encoding-fqn
       AS ENUM (
 	      'transit+json',
-	      'edn');
+	      'edn',
+	      'nippy');
 EXCEPTION
   when others then
 end $$;

src/document_storage/postgres/documents.clj

 (ns document-storage.postgres.documents
-  (:require [document-storage.postgres.encoding :as encoding]))
+  (:require [document-storage.postgres.encoding :as encoding]
+            [editscript.core :as escript]))
 
 
+(defn get-document-frame-by-version [storage document-id version repository]
+  (.run-query storage :get-document-frame-by-version
+              {:repository repository
+               :version version
+               :document-id document-id}))
+
+(defn get-document-frame-by-id [storage frame-id repository]
+  (.run-query storage :get-document-frame
+              {:repository repository
+               :id frame-id}))
+
+(defn get-last-document-frame [storage document-id repository]
+  (when-let [frame (.run-query storage :get-last-document-frame
+                             {:repository repository
+                              :document-id document-id})]
+    frame))
+
 (defn get-last-document-version [storage document-id repository]
-  (if-let [frame (.run-query storage :get-last-document-frame
-                               {:repository repository
-                                :document-id document-id})]
-    (:version frame)
-    -1))
+  (when-let [frame (get-last-document-frame storage document-id repository)]
+    (:version frame)))
+
+
+(defn restore-document [storage frame-id repository]
+  (when-let [frame (get-document-frame-by-id storage frame-id repository)]
+    (case (:data_type frame)
+      "key" (encoding/decode-document (:data frame))
+      "diff" (let [data (encoding/decode-document (:data frame))]
+               (escript/patch
+                       ;;;FIXME load document-frame by ID
+                (restore-document storage (:reference frame) repository)
+                (escript/edits->script data)))
+      (throw
+       (ex-info (str "Unknown data-type " (:data_type frame) " for frame " frame-id)
+                {:frame-id frame-id
+                 :frame frame})))))
 
 (defn load-document
   ([storage document-id repository]
-   (when-let [doc (.run-query storage :get-last-document-frame
-                              {:repository repository
-                               :document-id document-id})]
-     (encoding/decode-document (:data doc))))
+   (when-let [last-document-version (get-last-document-version storage document-id repository)]
+     (load-document storage document-id last-document-version repository)))
   ([storage document-id version repository]
-   (when-let [doc (.run-query storage :get-document-frame-by-version
-                              {:repository repository
-                               :version version
-                               :document-id document-id})]
-     (encoding/decode-document (:data doc)))))
+   (when-let [frame (get-document-frame-by-version storage document-id version repository)]
+     (restore-document storage (:id frame) repository))))
 
-(defn save-document [storage document-id document repository]
+(defn save-keyframe [storage document-id document repository]
   (let [doc (encoding/encode-document document)]
     ;;VALUES (:uuid, :document-id, :type, :encoding, :version, :reference, :data)
     (.run-query storage :create-document-frame!
                 {:repository repository
                  :document-id document-id
                  :type :ds-type/key
-                 :encoding :ds-encoding/transit+json
-                 :version (inc (get-last-document-version storage document-id repository))
+                 :encoding :ds-encoding/nippy
+                 :version 0
                  :reference nil
                  :data doc})))
 
 
+(defn save-pframe [storage document-id doc last-version repository]
+  (let [last-doc (load-document storage document-id last-version repository)
+        last-frame (get-document-frame-by-version storage document-id last-version repository)
+        diff (escript/diff last-doc doc)
+        diff-doc (encoding/encode-document (escript/get-edits diff))]
+    (.run-query storage :create-document-frame!
+                {:repository repository
+                 :document-id document-id
+                 :type :ds-type/diff
+                 :encoding :ds-encoding/nippy
+                 :version (inc last-version)
+                 :reference (:id last-frame) ;;;FIXME save ID in reference not version
+                 :data diff-doc})))
+
+(defn save-document [storage document-id document repository]
+  (if-let [last-version (get-last-document-version storage document-id repository)]
+    (save-pframe storage document-id document last-version repository)
+    (save-keyframe storage document-id document repository)))
+
+
 (defn delete-document [storage document-id repository]
     (.run-query storage :create-document-frame!
                 {:repository repository

src/document_storage/postgres/encoding.clj

 (ns document-storage.postgres.encoding
-  (:require [cognitect.transit :as transit])
+  (:require [cognitect.transit :as transit]
+            [taoensso.nippy :as nippy-compression :as nippy]
+            [taoensso.nippy.compression :as nippy-compression])
   (:import [java.io ByteArrayInputStream ByteArrayOutputStream]))
 
-(defn encode-document [document]
+(defn document->transit [document]
   (let [out (ByteArrayOutputStream.)
         writer (transit/writer out :json)]
     (transit/write writer document)
     (.toString out)))
 
-(defn decode-document [datastring]
+(defn transit->document [datastring]
   (let [in (ByteArrayInputStream. (.getBytes datastring))
         reader (transit/reader in :json)]
     (transit/read reader)))
+
+
+(defn encode-document [data]
+  (nippy/freeze data {:compression nippy-compression/lz4-compressor}))
+
+(defn decode-document [data]
+  (nippy/thaw data))

test/document_storage/postgres/core_test.clj

@@ -24,5 +24,6 @@
 (deftest simple-test
   (protocol-tests/test-save-document ds-atom))
 
+
 (deftest versioning-test
     (protocol-tests/test-versioning ds-atom))