Skip to content
Snippets Groups Projects
Commit b1a9fff3 authored by Bruno Burke's avatar Bruno Burke :hamburger:
Browse files

postgres: save documents as escript-diffs and keyframes, use nippy for serialization

parent 832bd10e
No related branches found
No related tags found
1 merge request!1Draft: Migration
Pipeline #91957 passed
......@@ -8,8 +8,10 @@
:dependencies [[org.clojure/clojure "1.10.1"]
[org.clojure/tools.logging "1.1.0"]
[com.fzakaria/slf4j-timbre "0.3.20"]
[com.taoensso/nippy "3.1.1"]
[clj-jgit "1.0.0-beta3"]
[fipp "0.6.21"]
[juji/editscript "0.5.7"]
[luminus-migrations "0.7.1"]
[com.cognitect/transit-clj "1.0.324"]
[conman "0.9.0"]
......
......@@ -8,7 +8,7 @@ CREATE TABLE :i:repository-table (
data_type :i:ds-type-fqn NOT NULL,
reference varchar(32) NULL,
created_at timestamptz(0) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"data" text NOT NULL,
"data" bytea NOT NULL,
"version" int8 NOT NULL DEFAULT 0,
data_encoding :i:ds-encoding-fqn NOT NULL,
CONSTRAINT
......
......@@ -29,7 +29,8 @@ DO $$DECLARE
:i:ds-encoding-fqn
AS ENUM (
'transit+json',
'edn');
'edn',
'nippy');
EXCEPTION
when others then
end $$;
(ns document-storage.postgres.documents
(:require [document-storage.postgres.encoding :as encoding]))
(:require [document-storage.postgres.encoding :as encoding]
[editscript.core :as escript]))
(defn get-document-frame-by-version [storage document-id version repository]
(.run-query storage :get-document-frame-by-version
{:repository repository
:version version
:document-id document-id}))
(defn get-document-frame-by-id [storage frame-id repository]
(.run-query storage :get-document-frame
{:repository repository
:id frame-id}))
(defn get-last-document-frame [storage document-id repository]
(when-let [frame (.run-query storage :get-last-document-frame
{:repository repository
:document-id document-id})]
frame))
(defn get-last-document-version [storage document-id repository]
(if-let [frame (.run-query storage :get-last-document-frame
{:repository repository
:document-id document-id})]
(:version frame)
-1))
(when-let [frame (get-last-document-frame storage document-id repository)]
(:version frame)))
(defn restore-document [storage frame-id repository]
(when-let [frame (get-document-frame-by-id storage frame-id repository)]
(case (:data_type frame)
"key" (encoding/decode-document (:data frame))
"diff" (let [data (encoding/decode-document (:data frame))]
(escript/patch
;;;FIXME load document-frame by ID
(restore-document storage (:reference frame) repository)
(escript/edits->script data)))
(throw
(ex-info (str "Unknown data-type " (:data_type frame) " for frame " frame-id)
{:frame-id frame-id
:frame frame})))))
(defn load-document
([storage document-id repository]
(when-let [doc (.run-query storage :get-last-document-frame
{:repository repository
:document-id document-id})]
(encoding/decode-document (:data doc))))
(when-let [last-document-version (get-last-document-version storage document-id repository)]
(load-document storage document-id last-document-version repository)))
([storage document-id version repository]
(when-let [doc (.run-query storage :get-document-frame-by-version
{:repository repository
:version version
:document-id document-id})]
(encoding/decode-document (:data doc)))))
(when-let [frame (get-document-frame-by-version storage document-id version repository)]
(restore-document storage (:id frame) repository))))
(defn save-document [storage document-id document repository]
(defn save-keyframe [storage document-id document repository]
(let [doc (encoding/encode-document document)]
;;VALUES (:uuid, :document-id, :type, :encoding, :version, :reference, :data)
(.run-query storage :create-document-frame!
{:repository repository
:document-id document-id
:type :ds-type/key
:encoding :ds-encoding/transit+json
:version (inc (get-last-document-version storage document-id repository))
:encoding :ds-encoding/nippy
:version 0
:reference nil
:data doc})))
(defn save-pframe [storage document-id doc last-version repository]
(let [last-doc (load-document storage document-id last-version repository)
last-frame (get-document-frame-by-version storage document-id last-version repository)
diff (escript/diff last-doc doc)
diff-doc (encoding/encode-document (escript/get-edits diff))]
(.run-query storage :create-document-frame!
{:repository repository
:document-id document-id
:type :ds-type/diff
:encoding :ds-encoding/nippy
:version (inc last-version)
:reference (:id last-frame) ;;;FIXME save ID in reference not version
:data diff-doc})))
(defn save-document [storage document-id document repository]
(if-let [last-version (get-last-document-version storage document-id repository)]
(save-pframe storage document-id document last-version repository)
(save-keyframe storage document-id document repository)))
(defn delete-document [storage document-id repository]
(.run-query storage :create-document-frame!
{:repository repository
......
(ns document-storage.postgres.encoding
(:require [cognitect.transit :as transit])
(:require [cognitect.transit :as transit]
[taoensso.nippy :as nippy-compression :as nippy]
[taoensso.nippy.compression :as nippy-compression])
(:import [java.io ByteArrayInputStream ByteArrayOutputStream]))
(defn encode-document [document]
(defn document->transit [document]
(let [out (ByteArrayOutputStream.)
writer (transit/writer out :json)]
(transit/write writer document)
(.toString out)))
(defn decode-document [datastring]
(defn transit->document [datastring]
(let [in (ByteArrayInputStream. (.getBytes datastring))
reader (transit/reader in :json)]
(transit/read reader)))
(defn encode-document [data]
(nippy/freeze data {:compression nippy-compression/lz4-compressor}))
(defn decode-document [data]
(nippy/thaw data))
......@@ -24,5 +24,6 @@
(deftest simple-test
(protocol-tests/test-save-document ds-atom))
(deftest versioning-test
(protocol-tests/test-versioning ds-atom))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment