Skip to content
Snippets Groups Projects
Commit 5d15ee34 authored by Bruno Burke's avatar Bruno Burke :hamburger:
Browse files

initial postgres storage version

parent c7ab9f24
No related branches found
No related tags found
1 merge request!1Draft: Migration
Pipeline #60863 failed
Showing with 446 additions and 8 deletions
......@@ -9,3 +9,4 @@ pom.xml.asc
/.nrepl-port
.hgignore
.hg/
profiles.clj
\ No newline at end of file
image: clojure:lein-2.7.1-alpine
services:
- postgres:12.2-alpine
variables:
POSTGRES_DB: docstorage
POSTGRES_USER: dbuser
POSTGRES_PASSWORD: "dbpassword"
POSTGRES_HOST_AUTH_METHOD: trust
stages:
- build
- test
......@@ -17,6 +26,8 @@ build:
- lein uberjar
test1:
variables:
DATABASE_URL: "jdbc:postgresql://localhost/docstorage?user=dbuser&password=dbpassword"
stage: test
script:
- lein test
......
(defproject document-storage "0.2.0"
(defproject document-storage "0.2.1"
:description "FIXME: write description"
:url "http://example.com/FIXME"
:license {:name "Eclipse Public License"
:url "http://www.eclipse.org/legal/epl-v10.html"}
:dependencies [[org.clojure/clojure "1.9.0"]
[org.clojure/tools.logging "0.4.0"]
:dependencies [[org.clojure/clojure "1.10.1"]
[org.clojure/tools.logging "1.1.0"]
[com.fzakaria/slf4j-timbre "0.3.20"]
[clj-jgit "1.0.0-beta3"]
[fipp "0.6.21"]])
[fipp "0.6.21"]
[luminus-migrations "0.7.1"]
[com.cognitect/transit-clj "1.0.324"]
[conman "0.9.0"]
[wwsoftware/clj-helper "0.0.3.5"]
[org.postgresql/postgresql "42.2.14"]]
:profiles {:test [:project/test :profiles/test]
:profiles/test {}
:project/test {:dependencies [[environ "1.2.0"]]
:plugins [[lein-environ "1.2.0"]]}
})
-- :name schema-exists? :? :*
-- :doc check if the dstorage schema exists
SELECT exists(select schema_name FROM information_schema.schemata WHERE schema_name = :storage-schema);
-- :name get-data :? :*
-- :doc retrieve a user given the id.
SELECT * FROM dstorage.users
WHERE id = :id
-- :name get-repositories :? :*
-- :doc retrieve a list of all repositories.
SELECT * FROM :i:repositories-table
-- :name get-document-frame :? :1
-- :doc retrieve a document-frame given the uuid.
SELECT * FROM :i:repository-table WHERE id = :id
-- :name get-document-versions :? :*
-- :doc retrieve all document-versions given the document-id.
SELECT version, data_type, created_at, id
FROM :i:repository-table
WHERE document_id = :document-id
ORDER BY version DESC
-- :name get-document-frame-by-version :? :1
-- :doc retrieve a document-frame given the document-id and version.
SELECT * FROM :i:repository-table WHERE document_id = :id AND version = :version
-- :name get-all-document-ids :? :*
-- :doc retrieve all document-ids
SELECT document_id
FROM :i:repository-table
GROUP BY document_id
-- :name create-document-frame! :! :n
-- :doc creates a new document frame record
INSERT INTO :i:repository-table
(id, document_id, data_type, data_encoding, version, reference, "data")
VALUES (:uuid, :document-id, :type, :encoding, :version, :reference, :data)
-- :name get-last-document-frame :? :1
-- :doc retrieve the last document-frame given the document-id.
SELECT * FROM :i:repository-table WHERE document_id = :document-id order by "version" desc limit 1
-- :name create-repository!
-- :command :execute
-- :result :raw
-- :doc creates a new repository table
CREATE TABLE :i:repository-table (
document_id varchar(64) NOT NULL,
id varchar(32) NOT NULL,
data_type :i:ds-type-fqn NOT NULL,
reference varchar(32) NULL,
created_at timestamptz(0) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"data" text NOT NULL,
"version" int8 NOT NULL DEFAULT 0,
data_encoding :i:ds-encoding-fqn NOT NULL,
CONSTRAINT
--~ (str "repo_" (:repository-name params) "_pk")
PRIMARY KEY (id),
CONSTRAINT
--~ (str "repo_" (:repository-name params) "_uk")
UNIQUE (version, document_id)
);
--;;
CREATE INDEX
--~ (str "repo_" (:repository-name params) "_document_id_idx")
ON :i:repository-table USING btree (document_id);
--;;
INSERT INTO :i:repositories-table
(id)
VALUES (:repository-name)
-- :name delete-repository!
-- :command :execute
-- :result :raw
-- :doc delete the repository table
DROP TABLE :i:repository-table;
--;;
DELETE FROM :i:repositories-table
WHERE id=:repository-name;
-- :name create-storage!
-- :command :execute
-- :result :raw
-- :doc creates the document storage tables and datatypes
CREATE SCHEMA IF NOT EXISTS :i:schema-name;
--;;
CREATE TABLE IF NOT EXISTS :i:repositories-table (
id varchar(32) NOT NULL,
created_at timestamptz(0) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT repositories_pk PRIMARY KEY (id));
--;;
DO $$DECLARE
BEGIN
CREATE TYPE
:i:ds-type-fqn
AS ENUM (
'keydiffcopydelete',
'key',
'diff',
'clone',
'delete');
EXCEPTION
when others then
end $$;
--;;
DO $$DECLARE
BEGIN
CREATE TYPE
:i:ds-encoding-fqn
AS ENUM (
'transit+json',
'edn');
EXCEPTION
when others then
end $$;
(ns document-storage.core
(:require [document-storage.git.core :as dsgit]))
(:require [document-storage.git.core :as dsgit]
[document-storage.postgres.core :as dspostgres]))
(defn make-git-ds [directory]
(.init-storage (dsgit/->git-storage {:dir directory})))
......@@ -3,7 +3,7 @@
[clojure.tools.logging :as log]
[clojure.java.io :as io]
[clojure.edn :refer [read-string]]
[document-storage.git.query :refer [query-fits? update-queries update-queries-remove-document]]
[document-storage.query :refer [query-fits? update-queries update-queries-remove-document]]
[document-storage.git.versioning :as versioning]
[document-storage.git.repository :refer [repositories init-repository valid-id?
default-repository
......@@ -106,7 +106,8 @@
:repositories @repositories
:cache (atom {})))
(log/error "Unable to initialize Document Storage - no config!")))
(list-repositories [this]
@repositories)
(load-document [this id repository]
(let [cached-repository (cached-repository? repository)]
(or
......
(ns document-storage.postgres.core
(:require [clojure.tools.logging :as log]
[clojure.java.io :as io]
[clojure.edn :refer [read-string]]
[clojure.set :refer [difference]]
[fipp.edn :refer [pprint] :rename {pprint fipp}]
[document-storage.protocol :as protocol]
[document-storage.postgres.db :as db]
[document-storage.postgres.documents :as documents]
[document-storage.query :refer [query-fits? update-queries
update-queries-remove-document]]
[conman.core :as conman]
[clj-helper.string :refer [get-unique-id]]))
(defprotocol db-storage
"Base Protocol for document storages"
(get-schema [this] "Return the db-schema name")
(create-repository [this repository] "Create a new repository")
(delete-repository [this repository] "Delete a repository")
(run-query [this query-key params] "Run a Query on a DB-based Storage"))
(defrecord postgres-storage [storage-id params connection queries]
db-storage
(get-schema [this]
(.-storage-id this))
(create-repository [this repository]
(.run-query this :create-repository! {:repository repository}))
(delete-repository [this repository]
(.run-query this :delete-repository! {:repository repository}))
(run-query [this query-key params]
(let [storage-schema (.get-schema this)
repository (:repository params)
repository-name (if (keyword? repository)
(name repository)
(str repository))
table-names {:schema-name storage-schema
:repository-name repository-name
:ds-encoding-fqn (str storage-schema
"."
"ds_encoding")
:ds-type-fqn (str storage-schema
"."
"ds_type")
:repositories-table (str storage-schema
"."
"repositories")
:repository-table (str storage-schema
"."
"repo_"
repository-name)
:uuid (get-unique-id "ds")}]
(conman/query (:queries this) query-key
(merge table-names params))))
protocol/storage
(init-storage [this]
(log/info "Init Postgres-Storage.")
(.run-query this :create-storage! {})
(let [repositories (set (map name (get (.-params this) :repositories [])))
existing-repos (set (map :id (.list-repositories this)))]
(log/info "Repositories: " repositories)
(log/info "Existing Repositories: " existing-repos)
(doseq [r (difference repositories existing-repos)]
(log/info "Create Repository: " r)
(.create-repository this r))))
(list-repositories [this]
(run-query this :get-repositories {}))
(load-document [this id repository]
(documents/load-document this id repository)
)
(get-document-size [this id repository]
(count (documents/load-document this id repository)))
(save-document [this id document repository]
(documents/save-document this id document repository))
(delete-document [this id repository]
(documents/delete-document this id repository))
(list-documents [this repository]
(into #{}
(map :document_id (run-query this :get-all-document-ids
{:repository repository}))))
(find-documents [this query repository]
(let [docs (.list-documents this :repository repository)]
(let [result (into {} (keep (fn [fname]
(let [doc (.load-document this fname :repository repository)]
(when (query-fits? doc query)
[fname doc]
)
))
docs))]
result)))
(shutdown [this]
(db/disconnect! (:connection this)))
(list-versions [this id repository]
(run-query this :get-document-versions {:repository repository
:document-id id}))
(load-version [this id version repository]
(documents/load-document this id version repository))
)
(defn make-postgres-ds [config]
(let [conn (db/connect! config)
id (str "ds" (name (or (:id config) "main")))
storage (->postgres-storage id {:repositories (:repositories config)} (:connection conn) (:queries conn))]
(.init-storage storage)
storage))
(ns document-storage.postgres.db
(:require
[conman.core :as conman]
[clojure.string :as str]
[hugsql.core :as hugsql]
[clojure.repl]
[clojure.java.jdbc :as jdbc]
[next.jdbc.prepare :as p]
[next.jdbc.result-set :as rs])
(:import
[org.postgresql.util PGobject]
[java.sql Timestamp]
[java.sql Date Timestamp PreparedStatement]))
(defn get-database-url [config]
(or (get config :jdbc-url)
(str "jdbc:postgresql://"
(get config :database-host) ":"
(get config :database-port) "/"
(get config :database-name) "?"
"user=" (get config :database-user) "&"
"password=" (get config :database-password))))
(defn connect! [config]
(let [connection (conman/connect! {:jdbc-url (get-database-url config)})]
{:connection connection
:queries (conman/bind-connection-map
connection
"sql/_ds_postgres.sql"
"sql/_ds_postgres_repository.sql"
"sql/_ds_postgres_storage.sql")}))
(defn disconnect! [connection]
(.close connection)
(conman/disconnect! connection))
(defn kw->pgenum [kw]
(let [type (-> (namespace kw)
(str/replace "-" "_"))
value (name kw)]
(doto (PGobject.)
(.setType type)
(.setValue value))))
(defn pg-param-type
[^PreparedStatement s ^long idx]
(if-let [md (.getParameterMetaData s)]
(or (.getParameterTypeName md idx)
(throw (ex-info "We could not obtain the column type name" {:got s :meta md})))
(throw (ex-info "We could not obtain metadata from the prepared statement" {:got s}))))
(defn <-pgobject
"Transform PGobject containing `json` or `jsonb` value to Clojure
data."
[^org.postgresql.util.PGobject v]
(let [type (.getType v)
value (.getValue v)]
(cond
(str/includes? type "ds_encoding") (keyword "ds-encoding" value)
(str/includes? type "ds_type") (keyword "ds-type" value)
:else value)))
(extend-protocol rs/ReadableColumn
org.postgresql.util.PGobject
(read-column-by-label [^org.postgresql.util.PGobject v _]
(<-pgobject v))
(read-column-by-index [^org.postgresql.util.PGobject v _2 _3]
(<-pgobject v)))
(extend-protocol p/SettableParameter
clojure.lang.Keyword
(set-parameter [^clojure.lang.Keyword v ^PreparedStatement ps ^long i]
(.setObject ps i (kw->pgenum v)))
java.util.Date
(set-parameter [^java.util.Date v ^PreparedStatement stmt ^long idx]
(.setObject stmt idx
(case (pg-param-type stmt idx)
"date" (Date. (.getTime v))
"timestamp" (Timestamp. (.getTime v))
"timestamptz" (Timestamp. (.getTime v))))))
(ns document-storage.postgres.documents
(:require [document-storage.postgres.encoding :as encoding]))
(defn get-last-document-version [storage document-id repository]
(if-let [frame (.run-query storage :get-last-document-frame
{:repository repository
:document-id document-id})]
(:version frame)
-1))
(defn load-document
([storage document-id repository]
(when-let [doc (.run-query storage :get-last-document-frame
{:repository repository
:document-id document-id})]
(encoding/decode-document (:data doc))))
([storage document-id version repository]
(when-let [doc (.run-query storage :get-document-frame-by-version
{:repository repository
:version version
:document-id document-id})]
(encoding/decode-document (:data doc)))))
(defn save-document [storage document-id document repository]
(let [doc (encoding/encode-document document)]
;;VALUES (:uuid, :document-id, :type, :encoding, :version, :reference, :data)
(.run-query storage :create-document-frame!
{:repository repository
:document-id document-id
:type :ds-type/key
:encoding :ds-encoding/transit+json
:version (inc (get-last-document-version storage document-id repository))
:reference nil
:data doc})))
(defn delete-document [storage document-id repository]
(.run-query storage :create-document-frame!
{:repository repository
:document-id document-id
:type :ds-type/delete
:encoding :ds-encoding/edn
:version (inc (get-last-document-version storage document-id repository))
:reference nil
:data ""}))
(ns document-storage.postgres.encoding
(:require [cognitect.transit :as transit])
(:import [java.io ByteArrayInputStream ByteArrayOutputStream]))
(defn encode-document [document]
(let [out (ByteArrayOutputStream.)
writer (transit/writer out :json)]
(transit/write writer document)
(.toString out)))
(defn decode-document [datastring]
(let [in (ByteArrayInputStream. (.getBytes datastring))
reader (transit/reader in :json)]
(transit/read reader)))
......@@ -7,6 +7,7 @@
(delete-document [ds id repository] "Delete a specific document in a repository")
(list-documents [ds repository] "List all documents in a repository")
(find-documents [ds query repository] "Find documents with a given query in a repository")
(list-repositories [ds] "List all repositories")
(list-versions [ds id repository] "List all version of a specific document in a repository")
(load-version [ds id version repository] "Load a specific document version in a repository")
(get-document-size [ds id repository] "Calculate the current document size")
......
(ns document-storage.git.query)
(ns document-storage.query)
(defn query-fits? [data query]
(reduce (fn [a b]
......
(ns document-storage.postgres.core-test
(:require [clojure.test :refer :all]
[document-storage.core :as ds]
[document-storage.postgres.core :as dspostgres]
[clojure.java.io :as io]
[fipp.edn :refer [pprint] :rename {pprint fipp}]
[document-storage.protocol :as dsprot]
[environ.core :refer [env]]))
(def ds-atom (atom nil))
(defn storage-fixture [f]
(println "ENV:" )
(println (env :database-url))
(reset! ds-atom
(dspostgres/make-postgres-ds
{:id :testing
:repositories [:testing]
:jdbc-url (env :database-url)}))
(f)
(.delete-repository @ds-atom :testing))
(use-fixtures :once storage-fixture)
(deftest simple-test
(testing "save basic structure and load it"
(let [structure {:a 7 :c [1 2 3 4 99 5 -4] :v 3}
id "simple-test1"
repo :testing]
(.save-document @ds-atom id structure repo)
(is (= structure
(.load-document @ds-atom id repo))))))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment