initial postgres storage version

.gitignore

@@ -9,3 +9,4 @@ pom.xml.asc
 /.nrepl-port
 .hgignore
 .hg/
+profiles.clj
\ No newline at end of file

.gitlab-ci.yml

 image: clojure:lein-2.7.1-alpine
 
+services:
+  - postgres:12.2-alpine
+
+variables:
+  POSTGRES_DB: docstorage
+  POSTGRES_USER: dbuser
+  POSTGRES_PASSWORD: "dbpassword"
+  POSTGRES_HOST_AUTH_METHOD: trust
+
 stages:
 - build
 - test
@@ -17,6 +26,8 @@ build:
     - lein uberjar
 
 test1:
+  variables:
+    DATABASE_URL: "jdbc:postgresql://localhost/docstorage?user=dbuser&password=dbpassword" 
   stage: test
   script:
     - lein test

project.clj

-(defproject document-storage "0.2.0"
+(defproject document-storage "0.2.1"
   :description "FIXME: write description"
   :url "http://example.com/FIXME"
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}
-  :dependencies [[org.clojure/clojure "1.9.0"]
-                 [org.clojure/tools.logging "0.4.0"]
+  :dependencies [[org.clojure/clojure "1.10.1"]
+                 [org.clojure/tools.logging "1.1.0"]
+                 [com.fzakaria/slf4j-timbre "0.3.20"]
                  [clj-jgit "1.0.0-beta3"]
-                 [fipp "0.6.21"]])
+                 [fipp "0.6.21"]
+                 [luminus-migrations "0.7.1"]
+                 [com.cognitect/transit-clj "1.0.324"]
+                 [conman "0.9.0"]
+                 [wwsoftware/clj-helper "0.0.3.5"]
+                 [org.postgresql/postgresql "42.2.14"]]
+  :profiles {:test [:project/test :profiles/test]
+             :profiles/test {}
+             :project/test {:dependencies [[environ "1.2.0"]]
+                            :plugins [[lein-environ "1.2.0"]]}
+             })
+

resources/sql/_ds_postgres.sql

+-- :name schema-exists? :? :*
+-- :doc check if the dstorage schema exists
+SELECT exists(select schema_name FROM information_schema.schemata WHERE schema_name = :storage-schema);
+
+-- :name get-data :? :*
+-- :doc retrieve a user given the id.
+SELECT * FROM dstorage.users
+ WHERE id = :id
+
+
+-- :name get-repositories :? :*
+-- :doc retrieve a list of all repositories.
+SELECT * FROM :i:repositories-table
+
+
+-- :name get-document-frame :? :1
+-- :doc retrieve a document-frame given the uuid.
+SELECT * FROM :i:repository-table WHERE id = :id
+
+-- :name get-document-versions :? :*
+-- :doc retrieve all document-versions given the document-id.
+SELECT version, data_type, created_at, id
+  FROM :i:repository-table
+ WHERE document_id = :document-id
+       ORDER BY version DESC
+
+-- :name get-document-frame-by-version :? :1
+-- :doc retrieve a document-frame given the document-id and version.
+SELECT * FROM :i:repository-table WHERE document_id = :id AND version = :version
+
+
+-- :name get-all-document-ids :? :*
+-- :doc retrieve all document-ids
+SELECT document_id
+  FROM :i:repository-table
+ GROUP BY document_id
+
+-- :name create-document-frame! :! :n
+-- :doc creates a new document frame record
+INSERT INTO :i:repository-table
+            (id, document_id, data_type, data_encoding, version,  reference, "data")
+            VALUES (:uuid, :document-id, :type, :encoding, :version, :reference, :data)
+
+-- :name get-last-document-frame :? :1
+-- :doc retrieve the last document-frame given the document-id.
+SELECT * FROM :i:repository-table WHERE document_id = :document-id order by "version" desc limit 1

resources/sql/_ds_postgres_repository.sql

+-- :name create-repository!
+-- :command :execute
+-- :result :raw
+-- :doc creates a new repository table
+CREATE TABLE :i:repository-table (
+	document_id varchar(64) NOT NULL,
+	id varchar(32) NOT NULL,
+	data_type :i:ds-type-fqn NOT NULL,
+	reference varchar(32) NULL,
+	created_at timestamptz(0) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+	"data" text NOT NULL,
+	"version" int8 NOT NULL DEFAULT 0,
+	data_encoding :i:ds-encoding-fqn NOT NULL,
+	CONSTRAINT
+  --~ (str "repo_" (:repository-name params) "_pk")
+  PRIMARY KEY (id),
+  CONSTRAINT
+  --~ (str "repo_" (:repository-name params) "_uk")
+  UNIQUE (version, document_id)
+);
+--;;
+CREATE INDEX
+  --~ (str "repo_" (:repository-name params) "_document_id_idx")
+  ON :i:repository-table USING btree (document_id);
+--;;
+INSERT INTO :i:repositories-table
+            (id)
+VALUES (:repository-name)
+
+
+-- :name delete-repository!
+-- :command :execute
+-- :result :raw
+-- :doc delete the repository table
+DROP TABLE :i:repository-table;
+--;;
+DELETE FROM :i:repositories-table
+ WHERE id=:repository-name;
+

resources/sql/_ds_postgres_storage.sql

+-- :name create-storage!
+-- :command :execute
+-- :result :raw
+-- :doc creates the document storage tables and datatypes
+CREATE SCHEMA IF NOT EXISTS :i:schema-name;
+--;;
+CREATE TABLE IF NOT EXISTS :i:repositories-table (
+	id varchar(32) NOT NULL,
+	created_at timestamptz(0) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+	CONSTRAINT repositories_pk PRIMARY KEY (id));
+--;;
+DO $$DECLARE
+  BEGIN
+    CREATE TYPE
+      :i:ds-type-fqn
+      AS ENUM (
+	      'keydiffcopydelete',
+	      'key',
+	      'diff',
+	      'clone',
+	      'delete');
+EXCEPTION
+  when others then
+end $$;
+--;;
+DO $$DECLARE
+  BEGIN
+    CREATE TYPE
+      :i:ds-encoding-fqn
+      AS ENUM (
+	      'transit+json',
+	      'edn');
+EXCEPTION
+  when others then
+end $$;

src/document_storage/core.clj

 (ns document-storage.core
-  (:require [document-storage.git.core :as dsgit]))
+  (:require [document-storage.git.core :as dsgit]
+            [document-storage.postgres.core :as dspostgres]))
 
 (defn make-git-ds [directory]
   (.init-storage (dsgit/->git-storage {:dir directory})))
+

src/document_storage/git/core.clj

@@ -3,7 +3,7 @@
             [clojure.tools.logging :as log]
             [clojure.java.io :as io]
             [clojure.edn :refer [read-string]]
-            [document-storage.git.query :refer [query-fits? update-queries update-queries-remove-document]]
+            [document-storage.query :refer [query-fits? update-queries update-queries-remove-document]]
             [document-storage.git.versioning :as versioning]
             [document-storage.git.repository :refer [repositories init-repository valid-id?
                                                  default-repository
@@ -106,7 +106,8 @@
                 :repositories @repositories
                 :cache (atom {})))
       (log/error "Unable to initialize Document Storage - no config!")))
-
+  (list-repositories [this]
+    @repositories)
   (load-document [this id repository]
     (let [cached-repository (cached-repository? repository)]
       (or

src/document_storage/postgres/core.clj

+(ns document-storage.postgres.core
+  (:require [clojure.tools.logging :as log]
+            [clojure.java.io :as io]
+            [clojure.edn :refer [read-string]]
+            [clojure.set :refer [difference]]
+            [fipp.edn :refer [pprint] :rename {pprint fipp}]
+            [document-storage.protocol :as protocol]
+            [document-storage.postgres.db :as db]
+            [document-storage.postgres.documents :as documents]
+            [document-storage.query :refer [query-fits? update-queries
+                                            update-queries-remove-document]]
+            [conman.core :as conman]
+            [clj-helper.string :refer [get-unique-id]]))
+
+(defprotocol db-storage
+  "Base Protocol for document storages"
+  (get-schema [this] "Return the db-schema name")
+  (create-repository [this repository] "Create a new repository")
+  (delete-repository [this repository] "Delete a repository")
+  (run-query [this query-key params] "Run a Query on a DB-based Storage"))
+
+(defrecord postgres-storage [storage-id params connection queries]
+  db-storage
+  (get-schema [this]
+    (.-storage-id this))
+  (create-repository [this repository]
+    (.run-query this :create-repository! {:repository repository}))
+  (delete-repository [this repository]
+    (.run-query this :delete-repository! {:repository repository}))
+  (run-query [this query-key params]
+    (let [storage-schema (.get-schema this)
+          repository (:repository params)
+          repository-name (if (keyword? repository)
+                            (name repository)
+                            (str repository))
+          table-names {:schema-name storage-schema
+                       :repository-name repository-name
+                       :ds-encoding-fqn (str storage-schema
+                                           "."
+                                           "ds_encoding")
+                       :ds-type-fqn (str storage-schema
+                                       "."
+                                       "ds_type")
+                       :repositories-table (str storage-schema
+                                                "."
+                                                "repositories")
+                       :repository-table (str storage-schema
+                                              "."
+                                              "repo_"
+                                              repository-name)
+                       :uuid (get-unique-id "ds")}]
+      (conman/query (:queries this) query-key
+                    (merge table-names params))))
+
+  protocol/storage
+  (init-storage [this]
+    (log/info "Init Postgres-Storage.")
+    (.run-query this :create-storage! {})
+    (let [repositories (set (map name (get (.-params this) :repositories [])))
+          existing-repos (set (map :id (.list-repositories this)))]
+      (log/info "Repositories: " repositories)
+      (log/info "Existing Repositories: " existing-repos)
+      (doseq [r (difference repositories existing-repos)]
+        (log/info "Create Repository: " r)
+        (.create-repository this r))))
+
+  (list-repositories [this]
+    (run-query this :get-repositories {}))
+
+  (load-document [this id repository]
+    (documents/load-document this id repository)
+    )
+
+  (get-document-size [this id repository]
+    (count (documents/load-document this id repository)))
+
+  (save-document [this id document repository]
+    (documents/save-document this id document repository))
+
+  (delete-document [this id repository]
+    (documents/delete-document this id repository))
+
+  (list-documents [this repository]
+    (into #{}
+          (map :document_id (run-query this :get-all-document-ids
+                                       {:repository repository}))))
+
+  (find-documents [this query repository]
+    (let [docs (.list-documents this :repository repository)]
+      (let [result (into {} (keep (fn [fname]
+                                    (let [doc (.load-document this fname :repository repository)]
+                                      (when (query-fits? doc query)
+                                        [fname doc]
+                                        )
+                                      ))
+                                  docs))]
+        result)))
+
+  (shutdown [this]
+    (db/disconnect! (:connection this)))
+
+  (list-versions [this id repository]
+    (run-query this :get-document-versions {:repository repository
+                                            :document-id id}))
+
+  (load-version [this id version repository]
+    (documents/load-document this id version repository))
+  )
+
+(defn make-postgres-ds [config]
+  (let [conn (db/connect! config)
+        id (str "ds" (name (or (:id config) "main")))
+        storage (->postgres-storage id {:repositories (:repositories config)} (:connection conn) (:queries conn))]
+    (.init-storage storage)
+    storage))

src/document_storage/postgres/db.clj

+(ns document-storage.postgres.db
+  (:require
+   [conman.core :as conman]
+   [clojure.string :as str]
+   [hugsql.core :as hugsql]
+   [clojure.repl]
+   [clojure.java.jdbc :as jdbc]
+   [next.jdbc.prepare :as p]
+   [next.jdbc.result-set :as rs])
+  (:import
+   [org.postgresql.util PGobject]
+   [java.sql Timestamp]
+   [java.sql Date Timestamp PreparedStatement]))
+
+(defn get-database-url [config]
+  (or (get config :jdbc-url)
+      (str "jdbc:postgresql://"
+           (get config :database-host) ":"
+           (get config :database-port) "/"
+           (get config :database-name) "?"
+           "user=" (get config :database-user) "&"
+           "password=" (get config :database-password))))
+
+
+(defn connect! [config]
+  (let [connection (conman/connect! {:jdbc-url (get-database-url config)})]
+    {:connection connection
+     :queries (conman/bind-connection-map
+               connection
+               "sql/_ds_postgres.sql"
+               "sql/_ds_postgres_repository.sql"
+               "sql/_ds_postgres_storage.sql")}))
+
+(defn disconnect! [connection]
+  (.close connection)
+  (conman/disconnect! connection))
+
+
+(defn kw->pgenum [kw]
+  (let [type (-> (namespace kw)
+                 (str/replace "-" "_"))
+        value (name kw)]
+    (doto (PGobject.)
+      (.setType type)
+      (.setValue value))))
+
+(defn pg-param-type
+  [^PreparedStatement s ^long idx]
+  (if-let [md (.getParameterMetaData s)]
+    (or (.getParameterTypeName md idx)
+        (throw (ex-info "We could not obtain the column type name" {:got s :meta md})))
+    (throw (ex-info "We could not obtain metadata from the prepared statement" {:got s}))))
+
+
+(defn <-pgobject
+  "Transform PGobject containing `json` or `jsonb` value to Clojure
+  data."
+  [^org.postgresql.util.PGobject v]
+  (let [type  (.getType v)
+        value (.getValue v)]
+    (cond
+      (str/includes? type "ds_encoding") (keyword "ds-encoding" value)
+      (str/includes? type "ds_type") (keyword "ds-type" value)
+      :else value)))
+
+(extend-protocol rs/ReadableColumn
+  org.postgresql.util.PGobject
+  (read-column-by-label [^org.postgresql.util.PGobject v _]
+    (<-pgobject v))
+  (read-column-by-index [^org.postgresql.util.PGobject v _2 _3]
+    (<-pgobject v)))
+
+(extend-protocol p/SettableParameter
+  clojure.lang.Keyword
+  (set-parameter [^clojure.lang.Keyword v ^PreparedStatement ps ^long i]
+    (.setObject ps i (kw->pgenum v)))
+  java.util.Date
+  (set-parameter [^java.util.Date v ^PreparedStatement stmt ^long idx]
+    (.setObject stmt idx
+                (case (pg-param-type stmt idx)
+                  "date" (Date. (.getTime v))
+                  "timestamp" (Timestamp. (.getTime v))
+                  "timestamptz" (Timestamp. (.getTime v))))))

src/document_storage/postgres/documents.clj

+(ns document-storage.postgres.documents
+  (:require [document-storage.postgres.encoding :as encoding]))
+
+
+(defn get-last-document-version [storage document-id repository]
+  (if-let [frame (.run-query storage :get-last-document-frame
+                               {:repository repository
+                                :document-id document-id})]
+    (:version frame)
+    -1))
+
+(defn load-document
+  ([storage document-id repository]
+   (when-let [doc (.run-query storage :get-last-document-frame
+                              {:repository repository
+                               :document-id document-id})]
+     (encoding/decode-document (:data doc))))
+  ([storage document-id version repository]
+   (when-let [doc (.run-query storage :get-document-frame-by-version
+                              {:repository repository
+                               :version version
+                               :document-id document-id})]
+     (encoding/decode-document (:data doc)))))
+
+(defn save-document [storage document-id document repository]
+  (let [doc (encoding/encode-document document)]
+    ;;VALUES (:uuid, :document-id, :type, :encoding, :version, :reference, :data)
+    (.run-query storage :create-document-frame!
+                {:repository repository
+                 :document-id document-id
+                 :type :ds-type/key
+                 :encoding :ds-encoding/transit+json
+                 :version (inc (get-last-document-version storage document-id repository))
+                 :reference nil
+                 :data doc})))
+
+
+(defn delete-document [storage document-id repository]
+    (.run-query storage :create-document-frame!
+                {:repository repository
+                 :document-id document-id
+                 :type :ds-type/delete
+                 :encoding :ds-encoding/edn
+                 :version (inc (get-last-document-version storage document-id repository))
+                 :reference nil
+                 :data ""}))

src/document_storage/postgres/encoding.clj

+(ns document-storage.postgres.encoding
+  (:require [cognitect.transit :as transit])
+  (:import [java.io ByteArrayInputStream ByteArrayOutputStream]))
+
+(defn encode-document [document]
+  (let [out (ByteArrayOutputStream.)
+        writer (transit/writer out :json)]
+    (transit/write writer document)
+    (.toString out)))
+
+(defn decode-document [datastring]
+  (let [in (ByteArrayInputStream. (.getBytes datastring))
+        reader (transit/reader in :json)]
+    (transit/read reader)))

src/document_storage/protocol.clj

@@ -7,6 +7,7 @@
   (delete-document [ds id repository] "Delete a specific document in a repository")
   (list-documents [ds repository] "List all documents in a repository")
   (find-documents [ds query repository] "Find documents with a given query in a repository")
+  (list-repositories [ds] "List all repositories")
   (list-versions [ds id repository] "List all version of a specific document in a repository")
   (load-version [ds id version repository] "Load a specific document version in a repository")
   (get-document-size [ds id repository] "Calculate the current document size")

src/document_storage/git/query.clj → src/document_storage/query.clj

-(ns document-storage.git.query)
+(ns document-storage.query)
 
 (defn query-fits? [data query]
   (reduce (fn [a b]

test/document_storage/postgres/core_test.clj

+(ns document-storage.postgres.core-test
+  (:require [clojure.test :refer :all]
+            [document-storage.core :as ds]
+            [document-storage.postgres.core :as dspostgres]
+            [clojure.java.io :as io]
+            [fipp.edn :refer [pprint] :rename {pprint fipp}]
+            [document-storage.protocol :as dsprot]
+            [environ.core :refer [env]]))
+
+(def ds-atom (atom nil))
+
+(defn storage-fixture [f]
+  (println "ENV:" )
+  (println (env :database-url))
+  (reset! ds-atom
+          (dspostgres/make-postgres-ds
+           {:id :testing
+            :repositories [:testing]
+            :jdbc-url (env :database-url)}))
+  (f)
+  (.delete-repository @ds-atom :testing))
+
+(use-fixtures :once storage-fixture)
+
+(deftest simple-test
+  (testing "save basic structure and load it"
+    (let [structure {:a 7 :c [1 2 3 4 99 5 -4] :v 3}
+          id "simple-test1"
+          repo :testing]
+      (.save-document @ds-atom id structure repo)
+      (is (= structure
+             (.load-document @ds-atom id repo))))))